diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst new file mode 100644 index 0000000000..416ce82ba6 --- /dev/null +++ b/CONTRIBUTING.rst @@ -0,0 +1,19 @@ +The source repository for this project can be found at: + + https://opendev.org/openstack/openstack-helm-infra + +Pull requests submitted through GitHub are not monitored. + +To start contributing to OpenStack, follow the steps in the contribution guide +to set up and use Gerrit: + + https://docs.openstack.org/contributors/code-and-documentation/quick-start.html + +Bugs should be filed on StoryBoard: + + https://storyboard.openstack.org/#!/project/openstack/openstack-helm-infra + +For more specific information about contributing to this repository, see the +openstack-helm infra contributor guide: + + https://docs.openstack.org/openstack-helm-infra/latest/contributor/contributing.html diff --git a/ca-clusterissuer/Chart.yaml b/ca-clusterissuer/Chart.yaml new file mode 100644 index 0000000000..79a4fe1f3d --- /dev/null +++ b/ca-clusterissuer/Chart.yaml @@ -0,0 +1,24 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: "1.0" +description: Certificate Issuer chart for OSH +home: https://cert-manager.io/ +name: ca-clusterissuer +version: 2024.2.0 +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit + version: ">= 0.1.0" +... diff --git a/ca-clusterissuer/templates/clusterissuer-ca.yaml b/ca-clusterissuer/templates/clusterissuer-ca.yaml new file mode 100644 index 0000000000..1f67d7b4a9 --- /dev/null +++ b/ca-clusterissuer/templates/clusterissuer-ca.yaml @@ -0,0 +1,28 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.clusterissuer }} +{{- $envAll := . }} +--- +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + name: {{ .Values.conf.ca.issuer.name }} + labels: +{{ tuple $envAll "cert-manager" "clusterissuer" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + ca: + secretName: {{ .Values.conf.ca.secret.name }} +... +{{- end }} diff --git a/ca-clusterissuer/templates/secret-ca.yaml b/ca-clusterissuer/templates/secret-ca.yaml new file mode 100644 index 0000000000..8c4472514c --- /dev/null +++ b/ca-clusterissuer/templates/secret-ca.yaml @@ -0,0 +1,26 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_ca }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.conf.ca.secret.name }} + namespace: {{ .Values.conf.ca.secret.namespace }} +data: + tls.crt: {{ .Values.conf.ca.secret.crt | default "" | b64enc }} + tls.key: {{ .Values.conf.ca.secret.key | default "" | b64enc }} +... +{{- end }} diff --git a/ca-clusterissuer/values.yaml b/ca-clusterissuer/values.yaml new file mode 100644 index 0000000000..a235a8d894 --- /dev/null +++ b/ca-clusterissuer/values.yaml @@ -0,0 +1,27 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +conf: + ca: + issuer: + name: ca-clusterissuer + secret: + name: secret-name + # Namespace where cert-manager is deployed. + namespace: cert-manager + crt: null + key: null + +manifests: + clusterissuer: true + secret_ca: true +... diff --git a/ca-issuer/Chart.yaml b/ca-issuer/Chart.yaml new file mode 100644 index 0000000000..8834a10f7e --- /dev/null +++ b/ca-issuer/Chart.yaml @@ -0,0 +1,24 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: "1.0" +description: Certificate Issuer chart for OSH +home: https://cert-manager.io/ +name: ca-issuer +version: 2024.2.0 +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit + version: ">= 0.1.0" +... diff --git a/ca-issuer/templates/issuer-ca.yaml b/ca-issuer/templates/issuer-ca.yaml new file mode 100644 index 0000000000..ee24c61910 --- /dev/null +++ b/ca-issuer/templates/issuer-ca.yaml @@ -0,0 +1,33 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.issuer }} +{{- $envAll := . }} +--- +{{- if semverCompare "< v1.0.0" .Values.cert_manager_version }} +apiVersion: cert-manager.io/v1alpha3 +{{- else }} +apiVersion: cert-manager.io/v1 +{{- end }} +kind: Issuer +metadata: + name: {{ .Values.conf.ca.issuer.name }} + namespace: {{ .Release.Namespace }} + labels: +{{ tuple $envAll "cert-manager" "issuer" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + ca: + secretName: {{ .Values.conf.ca.secret.name }} +... +{{- end }} diff --git a/ca-issuer/templates/secret-ca.yaml b/ca-issuer/templates/secret-ca.yaml new file mode 100644 index 0000000000..5261a1df36 --- /dev/null +++ b/ca-issuer/templates/secret-ca.yaml @@ -0,0 +1,26 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_ca }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.conf.ca.secret.name }} + namespace: {{ .Release.Namespace }} +data: + tls.crt: {{ .Values.conf.ca.secret.crt | default "" | b64enc }} + tls.key: {{ .Values.conf.ca.secret.key | default "" | b64enc }} +... +{{- end }} diff --git a/ca-issuer/values.yaml b/ca-issuer/values.yaml new file mode 100644 index 0000000000..a9a717126d --- /dev/null +++ b/ca-issuer/values.yaml @@ -0,0 +1,30 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +conf: + ca: + issuer: + name: ca-issuer + secret: + name: secret-name + crt: null + key: null + +# Default Version of jetstack/cert-manager being deployed. +# Starting at v1.0.0, api-version: cert-manager.io/v1 is used +# For previous apiVersion: cert-manager.io/v1alpha3, change to older version (such as v0.15.0) +cert_manager_version: v1.0.0 + +manifests: + issuer: true + secret_ca: true +... diff --git a/ceph-adapter-rook/Chart.yaml b/ceph-adapter-rook/Chart.yaml new file mode 100644 index 0000000000..7c3191fd5a --- /dev/null +++ b/ceph-adapter-rook/Chart.yaml @@ -0,0 +1,24 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: v1.0.0 +description: OpenStack-Helm Ceph Adapter Rook +name: ceph-adapter-rook +version: 2024.2.0 +home: https://github.com/ceph/ceph +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit + version: ">= 0.1.0" +... diff --git a/ceph-adapter-rook/README.md b/ceph-adapter-rook/README.md new file mode 100644 index 0000000000..7a3e5b5579 --- /dev/null +++ b/ceph-adapter-rook/README.md @@ -0,0 +1,18 @@ +# Summary +This is the minimal set of templates necessary to make the rest +of Openstack-Helm charts work with Ceph clusters managed by the +Rook operator. Rook operator not only deploys Ceph clusters but +also provides convenience when interfacing with those clusters +via CRDs which can be used for managing pools/keys/users etc. +However Openstack-Helm charts do not utilize Rook CRDs but instead +manage Ceph assets like pools/keyrings/users/buckets etc. by means +of running bootstrap scripts. Before using Openstack-Helm charts we +have to provision a minimal set of assets like Ceph admin key and +Ceph client config. + +# Usage +helm upgrade --install ceph-adapter-rook ./ceph-adapter-rook \ + --namespace=openstack +``` + +Once all the jobs are finished you can deploy other Openstack-Helm charts. diff --git a/ceph-adapter-rook/templates/bin/_config-manager.sh.tpl b/ceph-adapter-rook/templates/bin/_config-manager.sh.tpl new file mode 100644 index 0000000000..3e7816fcf5 --- /dev/null +++ b/ceph-adapter-rook/templates/bin/_config-manager.sh.tpl @@ -0,0 +1,26 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +{{- $envAll := . }} + +ENDPOINTS=$(kubectl --namespace ${CEPH_CLUSTER_NAMESPACE} get configmap rook-ceph-mon-endpoints -o jsonpath='{.data.data}' | sed 's/.=//g') + +kubectl get cm ${CEPH_CONF_ETC} -n ${DEPLOYMENT_NAMESPACE} -o yaml | \ + sed "s#mon_host.*#mon_host = ${ENDPOINTS}#g" | \ + kubectl apply -f - + +kubectl get cm ${CEPH_CONF_ETC} -n ${DEPLOYMENT_NAMESPACE} -o yaml diff --git a/ceph-adapter-rook/templates/bin/_key-manager.sh.tpl b/ceph-adapter-rook/templates/bin/_key-manager.sh.tpl new file mode 100644 index 0000000000..1b10ab8f94 --- /dev/null +++ b/ceph-adapter-rook/templates/bin/_key-manager.sh.tpl @@ -0,0 +1,44 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +{{- $envAll := . }} + +# We expect rook-ceph-tools pod to be up and running +ROOK_CEPH_TOOLS_POD=$(kubectl -n ${CEPH_CLUSTER_NAMESPACE} get pods --no-headers | awk '/rook-ceph-tools/{print $1}') +CEPH_ADMIN_KEY=$(kubectl -n ${CEPH_CLUSTER_NAMESPACE} exec ${ROOK_CEPH_TOOLS_POD} -- ceph auth ls | grep -A1 "client.admin" | awk '/key:/{print $2}') + +ceph_activate_namespace() { + kube_namespace=$1 + secret_type=$2 + secret_name=$3 + ceph_key=$4 + { + cat <= 0.1.0" +... diff --git a/ceph-client/templates/bin/_bootstrap.sh.tpl b/ceph-client/templates/bin/_bootstrap.sh.tpl new file mode 100644 index 0000000000..6452d0a073 --- /dev/null +++ b/ceph-client/templates/bin/_bootstrap.sh.tpl @@ -0,0 +1,18 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +{{ .Values.bootstrap.script | default "echo 'Not Enabled'" }} diff --git a/ceph-client/templates/bin/_helm-tests.sh.tpl b/ceph-client/templates/bin/_helm-tests.sh.tpl new file mode 100755 index 0000000000..fa40068d27 --- /dev/null +++ b/ceph-client/templates/bin/_helm-tests.sh.tpl @@ -0,0 +1,429 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +function check_cluster_status() { + echo "#### Start: Checking Ceph cluster status ####" + ceph_status_output=$(ceph -s -f json | jq -r '.health') + ceph_health_status=$(echo $ceph_status_output | jq -r '.status') + + if [ "x${ceph_health_status}" == "xHEALTH_OK" ]; then + echo "Ceph status is HEALTH_OK" + else + echo "Ceph cluster status is not HEALTH_OK, checking PG states" + pg_validation + fi +} + +function check_recovery_flags() { + echo "### Start: Checking for flags that will prevent recovery" + + # Ensure there are no flags set that will prevent recovery of degraded PGs + if [[ $(ceph osd stat | grep "norecover\|nobackfill\|norebalance") ]]; then + ceph osd stat + echo "Flags are set that prevent recovery of degraded PGs" + exit 1 + fi +} + +function check_osd_count() { + echo "#### Start: Checking OSD count ####" + noup_flag=$(ceph osd stat | awk '/noup/ {print $2}') + osd_stat=$(ceph osd stat -f json-pretty) + num_osd=$(awk '/"num_osds"/{print $2}' <<< "$osd_stat" | cut -d, -f1) + num_in_osds=$(awk '/"num_in_osds"/{print $2}' <<< "$osd_stat" | cut -d, -f1) + num_up_osds=$(awk '/"num_up_osds"/{print $2}' <<< "$osd_stat" | cut -d, -f1) + + MIN_OSDS=$((${num_osd}*$REQUIRED_PERCENT_OF_OSDS/100)) + if [ ${MIN_OSDS} -lt 1 ]; then + MIN_OSDS=1 + fi + + if [ "${noup_flag}" ]; then + osd_status=$(ceph osd dump -f json | jq -c '.osds[] | .state') + count=0 + for osd in $osd_status; do + if [[ "$osd" == *"up"* || "$osd" == *"new"* ]]; then + ((count=count+1)) + fi + done + echo "Caution: noup flag is set. ${count} OSDs in up/new state. Required number of OSDs: ${MIN_OSDS}." + if [ $MIN_OSDS -gt $count ]; then + exit 1 + fi + else + if [ "${num_osd}" -eq 0 ]; then + echo "There are no osds in the cluster" + exit 1 + elif [ "${num_in_osds}" -ge "${MIN_OSDS}" ] && [ "${num_up_osds}" -ge "${MIN_OSDS}" ]; then + echo "Required number of OSDs (${MIN_OSDS}) are UP and IN status" + else + echo "Required number of OSDs (${MIN_OSDS}) are NOT UP and IN status. Cluster shows OSD count=${num_osd}, UP=${num_up_osds}, IN=${num_in_osds}" + exit 1 + fi + fi +} + +function check_failure_domain_count_per_pool() { + echo "#### Start: Checking failure domain count per pool ####" + pools=$(ceph osd pool ls) + for pool in ${pools} + do + crush_rule=$(ceph osd pool get ${pool} crush_rule | awk '{print $2}') + bucket_type=$(ceph osd crush rule dump ${crush_rule} | grep '"type":' | awk -F'"' 'NR==2 {print $4}') + num_failure_domains=$(ceph osd tree | grep ${bucket_type} | wc -l) + pool_replica_size=$(ceph osd pool get ${pool} size | awk '{print $2}') + if [[ ${num_failure_domains} -ge ${pool_replica_size} ]]; then + echo "--> Info: Pool ${pool} is configured with enough failure domains ${num_failure_domains} to satisfy pool replica size ${pool_replica_size}" + else + echo "--> Error : Pool ${pool} is NOT configured with enough failure domains ${num_failure_domains} to satisfy pool replica size ${pool_replica_size}" + exit 1 + fi + done +} + +function mgr_validation() { + echo "#### Start: MGR validation ####" + mgr_dump=$(ceph mgr dump -f json-pretty) + echo "Checking for ${MGR_COUNT} MGRs" + + mgr_avl=$(echo ${mgr_dump} | jq -r '.["available"]') + + if [ "x${mgr_avl}" == "xtrue" ]; then + mgr_active=$(echo ${mgr_dump} | jq -r '.["active_name"]') + echo "Out of ${MGR_COUNT}, 1 MGR is active" + + # Now lets check for standby managers + mgr_stdby_count=$(echo ${mgr_dump} | jq -r '.["standbys"]' | jq length) + + #Total MGR Count - 1 Active = Expected MGRs + expected_standbys=$(( MGR_COUNT -1 )) + + if [ $mgr_stdby_count -eq $expected_standbys ] + then + echo "Cluster has 1 Active MGR, $mgr_stdby_count Standbys MGR" + else + echo "Warning. Cluster Standbys MGR: Expected count= $expected_standbys Available=$mgr_stdby_count" + echo "If this is not expected behavior, please investigate and take some additional actions." + fi + + else + echo "No Active Manager found, Expected 1 MGR to be active out of ${MGR_COUNT}" + retcode=1 + fi + + if [ "x${retcode}" == "x1" ] + then + exit 1 + fi +} + +function pool_validation() { + + echo "#### Start: Checking Ceph pools ####" + + echo "From env variables, RBD pool replication count is: ${RBD}" + + # Assuming all pools have same replication count as RBD + # If RBD replication count is greater then 1, POOLMINSIZE should be 1 less then replication count + # If RBD replication count is not greate then 1, then POOLMINSIZE should be 1 + + if [ ${RBD} -gt 1 ]; then + EXPECTED_POOLMINSIZE=$[${RBD}-1] + else + EXPECTED_POOLMINSIZE=1 + fi + + echo "EXPECTED_POOLMINSIZE: ${EXPECTED_POOLMINSIZE}" + + expectedCrushRuleId="" + nrules=$(echo ${OSD_CRUSH_RULE_DUMP} | jq length) + c=$[nrules-1] + for n in $(seq 0 ${c}) + do + osd_crush_rule_obj=$(echo ${OSD_CRUSH_RULE_DUMP} | jq -r .[${n}]) + + name=$(echo ${osd_crush_rule_obj} | jq -r .rule_name) + echo "Expected Crushrule: ${EXPECTED_CRUSHRULE}, Pool Crushmap: ${name}" + + if [ "x${EXPECTED_CRUSHRULE}" == "x${name}" ]; then + expectedCrushRuleId=$(echo ${osd_crush_rule_obj} | jq .rule_id) + echo "Checking against rule: id: ${expectedCrushRuleId}, name:${name}" + else + echo "Didn't match" + fi + done + echo "Checking cluster for size:${RBD}, min_size:${EXPECTED_POOLMINSIZE}, crush_rule:${EXPECTED_CRUSHRULE}, crush_rule_id:${expectedCrushRuleId}" + + npools=$(echo ${OSD_POOLS_DETAILS} | jq length) + i=$[npools - 1] + for n in $(seq 0 ${i}) + do + pool_obj=$(echo ${OSD_POOLS_DETAILS} | jq -r ".[${n}]") + + size=$(echo ${pool_obj} | jq -r .size) + min_size=$(echo ${pool_obj} | jq -r .min_size) + pg_num=$(echo ${pool_obj} | jq -r .pg_num) + pg_placement_num=$(echo ${pool_obj} | jq -r .pg_placement_num) + crush_rule=$(echo ${pool_obj} | jq -r .crush_rule) + name=$(echo ${pool_obj} | jq -r .pool_name) + pg_autoscale_mode=$(echo ${pool_obj} | jq -r .pg_autoscale_mode) + if [[ "${ENABLE_AUTOSCALER}" == "true" ]]; then + if [[ "${pg_autoscale_mode}" != "on" ]]; then + echo "pg autoscaler not enabled on ${name} pool" + exit 1 + fi + fi + if [[ $(ceph mon versions | awk '/version/{print $3}' | cut -d. -f1) -ge 14 ]]; then + if [ "x${size}" != "x${RBD}" ] || [ "x${min_size}" != "x${EXPECTED_POOLMINSIZE}" ] \ + || [ "x${crush_rule}" != "x${expectedCrushRuleId}" ]; then + echo "Pool ${name} has incorrect parameters!!! Size=${size}, Min_Size=${min_size}, Rule=${crush_rule}, PG_Autoscale_Mode=${pg_autoscale_mode}" + exit 1 + else + echo "Pool ${name} seems configured properly. Size=${size}, Min_Size=${min_size}, Rule=${crush_rule}, PG_Autoscale_Mode=${pg_autoscale_mode}" + fi + else + if [ "x${size}" != "x${RBD}" ] || [ "x${min_size}" != "x${EXPECTED_POOLMINSIZE}" ] \ + || [ "x${pg_num}" != "x${pg_placement_num}" ] || [ "x${crush_rule}" != "x${expectedCrushRuleId}" ]; then + echo "Pool ${name} has incorrect parameters!!! Size=${size}, Min_Size=${min_size}, PG=${pg_num}, PGP=${pg_placement_num}, Rule=${crush_rule}" + exit 1 + else + echo "Pool ${name} seems configured properly. Size=${size}, Min_Size=${min_size}, PG=${pg_num}, PGP=${pg_placement_num}, Rule=${crush_rule}" + fi + fi + done +} + +function pool_failuredomain_validation() { + echo "#### Start: Checking Pools are configured with specific failure domain ####" + + expectedCrushRuleId="" + nrules=$(echo ${OSD_CRUSH_RULE_DUMP} | jq length) + c=$[nrules-1] + for n in $(seq 0 ${c}) + do + osd_crush_rule_obj=$(echo ${OSD_CRUSH_RULE_DUMP} | jq -r .[${n}]) + + name=$(echo ${osd_crush_rule_obj} | jq -r .rule_name) + + if [ "x${EXPECTED_CRUSHRULE}" == "x${name}" ]; then + expectedCrushRuleId=$(echo ${osd_crush_rule_obj} | jq .rule_id) + echo "Checking against rule: id: ${expectedCrushRuleId}, name:${name}" + fi + done + + echo "Checking OSD pools are configured with Crush rule name:${EXPECTED_CRUSHRULE}, id:${expectedCrushRuleId}" + + npools=$(echo ${OSD_POOLS_DETAILS} | jq length) + i=$[npools-1] + for p in $(seq 0 ${i}) + do + pool_obj=$(echo ${OSD_POOLS_DETAILS} | jq -r ".[${p}]") + + pool_crush_rule_id=$(echo $pool_obj | jq -r .crush_rule) + pool_name=$(echo $pool_obj | jq -r .pool_name) + + if [ "x${pool_crush_rule_id}" == "x${expectedCrushRuleId}" ]; then + echo "--> Info: Pool ${pool_name} is configured with the correct rule ${pool_crush_rule_id}" + else + echo "--> Error : Pool ${pool_name} is NOT configured with the correct rule ${pool_crush_rule_id}" + exit 1 + fi + done +} + +function check_transient_pgs_file() { + current_time=$1 + pg_failed_list=() + + # Remove the lines NOT having the word "current" as these are the old + # PGs that are no longer in transition. + sed -i '/current/!d' ${transient_pgs_file} + + # For all remaining lines (PGs currently inactive), check for PGs which + # are older than the limit. + IFS=$'\n' read -d '' -r -a lines < ${transient_pgs_file} || true + for pg_data in "${lines[@]}"; do + pg=$(echo ${pg_data} | awk '{print $1}') + pg_ts=$(echo ${pg_data} | awk '{print $2}') + if [[ $((${current_time} - ${pg_ts})) -gt ${pg_inactive_timeout} ]]; then + pg_failed_list+=("${pg}") + fi + done + + # Remove the current designation for all PGs, as we no longer need it + # for this check. + sed -i 's/ current//g' ${transient_pgs_file} + + cat ${transient_pgs_file} + if [[ ${#pg_failed_list[@]} -gt 0 ]]; then + echo "The following PGs have been in a transient state for longer than ${pg_inactive_timeout} seconds:" + echo ${pg_failed_list[*]} + exit 1 + fi +} + +function update_transient_pgs_file() { + pg=$1 + current_ts=$2 + + pg_data=$(grep "${pg} " ${transient_pgs_file} || true) + if [[ "${pg_data}" == "" ]]; then + echo "${pg} ${current_ts} current" >> ${transient_pgs_file} + else + # Add the word "current" to the end of the line which has this PG + sed -i '/^'"${pg} "'/s/$/ current/' ${transient_pgs_file} + fi +} + +function check_transient_pgs() { + local -n pg_array=$1 + + # Use a temporary transient PGs file to track the amount of time PGs + # are spending in a transitional state. + now=$(date +%s) + for pg in "${pg_array[@]}"; do + update_transient_pgs_file ${pg} ${now} + done + check_transient_pgs_file ${now} +} + +function check_pgs() { + pgs_transitioning=false + + ceph --cluster ${CLUSTER} pg dump_stuck inactive -f json-pretty > ${stuck_pgs_file} + + # Check if there are any stuck PGs, which could indicate a serious problem + # if it does not resolve itself soon. + stuck_pgs=(`cat ${stuck_pgs_file} | awk -F "\"" '/pgid/{print $4}'`) + if [[ ${#stuck_pgs[*]} -gt 0 ]]; then + # We have at least one stuck pg + echo "Some PGs are stuck: " + echo ${stuck_pgs[*]} + # Not a critical error - yet + pgs_transitioning=true + + # Check to see if any transitioning PG has been stuck for too long + check_transient_pgs stuck_pgs + else + # Examine the PGs that have non-active states. Consider those PGs that + # are in a "premerge" state to be similar to active. "premerge" PGs may + # stay in that state for several minutes, and this is considered ok. + ceph --cluster ${CLUSTER} pg ls -f json-pretty | grep '"pgid":\|"state":' | grep -v -E "active|premerge" | grep -B1 '"state":' > ${inactive_pgs_file} || true + + # If the inactive pgs file is non-empty, there are some inactive pgs in the cluster. + inactive_pgs=(`cat ${inactive_pgs_file} | awk -F "\"" '/pgid/{print $4}'`) + echo "This is the list of inactive pgs in the cluster: " + echo ${inactive_pgs[*]} + + echo "Checking to see if the cluster is rebalancing or recovering some PG's..." + + # Check for PGs that are down. These are critical errors. + down_pgs=(`cat ${inactive_pgs_file} | grep -B1 'down' | awk -F "\"" '/pgid/{print $4}'`) + if [[ ${#down_pgs[*]} -gt 0 ]]; then + # Some PGs could be down. This is really bad situation and test must fail. + echo "Some PGs are down: " + echo ${down_pgs[*]} + echo "This is critical error, exiting. " + exit 1 + fi + + # Check for PGs that are in some transient state due to rebalancing, + # peering or backfilling. If we see other states which are not in the + # following list of states, then we likely have a problem and need to + # exit. + transient_states='peer|recover|activating|creating|unknown' + non_transient_pgs=(`cat ${inactive_pgs_file} | grep '"state":' | grep -v -E "${transient_states}" || true`) + if [[ ${#non_transient_pgs[*]} -gt 0 ]]; then + # Some PGs could be inactive and not peering. Better we fail. + echo "We don't have down/stuck PGs, but we have some inactive pgs that" + echo "are not in the list of allowed transient states: " + pg_list=(`sed -n '/peer\|recover\|activating\|creating\|unknown/{s/.*//;x;d;};x;p;${x;p;}' ${inactive_pgs_file} | sed '/^$/d' | awk -F "\"" '/pgid/{print $4}'`) + echo ${pg_list[*]} + echo ${non_transient_pgs[*]} + # Critical error. Fail/exit the script + exit 1 + fi + + # Check and note which PGs are in a transient state. This script + # will allow these transient states for a period of time + # (time_between_retries * max_retries seconds). + transient_pgs=(`cat ${inactive_pgs_file} | grep -B1 -E "${transient_states}" | awk -F "\"" '/pgid/{print $4}'`) + if [[ ${#transient_pgs[*]} -gt 0 ]]; then + # Some PGs are not in an active state but peering and/or cluster is recovering + echo "Some PGs are peering and/or cluster is recovering: " + echo ${transient_pgs[*]} + echo "This is normal but will wait a while to verify the PGs are not stuck in a transient state." + # not critical, just wait + pgs_transitioning=true + + # Check to see if any transitioning PG has been stuck for too long + check_transient_pgs transient_pgs + fi + fi +} + +function pg_validation() { + retries=0 + time_between_retries=3 + max_retries=60 + pg_inactive_timeout=30 + pgs_transitioning=false + stuck_pgs_file=$(mktemp -p /tmp) + inactive_pgs_file=$(mktemp -p /tmp) + transient_pgs_file=$(mktemp -p /tmp) + + # Check this over a period of retries. Fail/stop if any critical errors found. + while check_pgs && [[ "${pgs_transitioning}" == "true" ]] && [[ retries -lt ${max_retries} ]]; do + echo "Sleep for a bit waiting on the pg(s) to become active/unstuck..." + sleep ${time_between_retries} + ((retries=retries+1)) + done + + # Check if transitioning PGs have gone active after retries have expired + if [[ retries -ge ${max_retries} ]]; then + ((timeout_sec=${time_between_retries}*${max_retries})) + echo "Some PGs have not become active after ${timeout_sec} seconds. Exiting..." + # This is ok, as the autoscaler might still be adjusting the PGs. + fi +} + +function check_ceph_osd_crush_weight(){ + OSDS_WITH_ZERO_WEIGHT=(`ceph --cluster ${CLUSTER} osd df -f json-pretty | awk -F"[, ]*" '/"crush_weight":/{if ($3 == 0) print $3}'`) + if [[ ${#OSDS_WITH_ZERO_WEIGHT[*]} -eq 0 ]]; then + echo "All OSDs from namespace have crush weight!" + else + echo "OSDs from namespace have zero crush weight" + exit 1 + fi +} + +check_osd_count +mgr_validation + +OSD_POOLS_DETAILS=$(ceph osd pool ls detail -f json-pretty) +OSD_CRUSH_RULE_DUMP=$(ceph osd crush rule dump -f json-pretty) +PG_STAT=$(ceph pg stat -f json-pretty) + +ceph -s +pg_validation +pool_validation +pool_failuredomain_validation +check_failure_domain_count_per_pool +check_cluster_status +check_recovery_flags +check_ceph_osd_crush_weight diff --git a/ceph-client/templates/bin/_init-dirs.sh.tpl b/ceph-client/templates/bin/_init-dirs.sh.tpl new file mode 100644 index 0000000000..a6b59075b4 --- /dev/null +++ b/ceph-client/templates/bin/_init-dirs.sh.tpl @@ -0,0 +1,44 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +export LC_ALL=C +: "${HOSTNAME:=$(uname -n)}" +: "${MGR_NAME:=${HOSTNAME}}" +: "${MDS_NAME:=mds-${HOSTNAME}}" +: "${MDS_BOOTSTRAP_KEYRING:=/var/lib/ceph/bootstrap-mds/${CLUSTER}.keyring}" +: "${OSD_BOOTSTRAP_KEYRING:=/var/lib/ceph/bootstrap-osd/${CLUSTER}.keyring}" + +for keyring in ${OSD_BOOTSTRAP_KEYRING} ${MDS_BOOTSTRAP_KEYRING}; do + mkdir -p "$(dirname "$keyring")" +done + +# Let's create the ceph directories +for DIRECTORY in mds tmp mgr crash; do + mkdir -p "/var/lib/ceph/${DIRECTORY}" +done + +# Create socket directory +mkdir -p /run/ceph + +# Create the MDS directory +mkdir -p "/var/lib/ceph/mds/${CLUSTER}-${MDS_NAME}" + +# Create the MGR directory +mkdir -p "/var/lib/ceph/mgr/${CLUSTER}-${MGR_NAME}" + +# Adjust the owner of all those directories +chown -R ceph. /run/ceph/ /var/lib/ceph/* diff --git a/ceph-client/templates/bin/mds/_start.sh.tpl b/ceph-client/templates/bin/mds/_start.sh.tpl new file mode 100644 index 0000000000..15eb4948ad --- /dev/null +++ b/ceph-client/templates/bin/mds/_start.sh.tpl @@ -0,0 +1,86 @@ +#!/bin/bash +set -ex +export LC_ALL=C +: "${HOSTNAME:=$(uname -n)}" +: "${CEPHFS_CREATE:=0}" +: "${CEPHFS_NAME:=cephfs}" +: "${CEPHFS_DATA_POOL:=${CEPHFS_NAME}_data}" +: "${CEPHFS_DATA_POOL_PG:=8}" +: "${CEPHFS_METADATA_POOL:=${CEPHFS_NAME}_metadata}" +: "${CEPHFS_METADATA_POOL_PG:=8}" +: "${MDS_NAME:=mds-${HOSTNAME}}" +: "${ADMIN_KEYRING:=/etc/ceph/${CLUSTER}.client.admin.keyring}" +: "${MDS_KEYRING:=/var/lib/ceph/mds/${CLUSTER}-${MDS_NAME}/keyring}" +: "${MDS_BOOTSTRAP_KEYRING:=/var/lib/ceph/bootstrap-mds/${CLUSTER}.keyring}" +: "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}" + +{{ include "helm-toolkit.snippets.mon_host_from_k8s_ep" . }} + +if [[ ! -e ${CEPH_CONF}.template ]]; then + echo "ERROR- ${CEPH_CONF}.template must exist; get it from your existing mon" + exit 1 +else + ENDPOINT=$(mon_host_from_k8s_ep "${NAMESPACE}" ceph-mon-discovery) + if [[ "${ENDPOINT}" == "" ]]; then + /bin/sh -c -e "cat ${CEPH_CONF}.template | tee ${CEPH_CONF}" || true + else + /bin/sh -c -e "cat ${CEPH_CONF}.template | sed 's#mon_host.*#mon_host = ${ENDPOINT}#g' | tee ${CEPH_CONF}" || true + fi +fi + +# Check to see if we are a new MDS +if [ ! -e "${MDS_KEYRING}" ]; then + + if [ -e "${ADMIN_KEYRING}" ]; then + KEYRING_OPT=(--name client.admin --keyring "${ADMIN_KEYRING}") + elif [ -e "${MDS_BOOTSTRAP_KEYRING}" ]; then + KEYRING_OPT=(--name client.bootstrap-mds --keyring "${MDS_BOOTSTRAP_KEYRING}") + else + echo "ERROR- Failed to bootstrap MDS: could not find admin or bootstrap-mds keyring. You can extract it from your current monitor by running 'ceph auth get client.bootstrap-mds -o ${MDS_BOOTSTRAP_KEYRING}" + exit 1 + fi + + timeout 10 ceph --cluster "${CLUSTER}" "${KEYRING_OPT[@]}" health || exit 1 + + # Generate the MDS key + ceph --cluster "${CLUSTER}" "${KEYRING_OPT[@]}" auth get-or-create "mds.${MDS_NAME}" osd 'allow rwx' mds 'allow' mon 'allow profile mds' -o "${MDS_KEYRING}" + chown ceph. "${MDS_KEYRING}" + chmod 600 "${MDS_KEYRING}" + +fi + +# NOTE (leseb): having the admin keyring is really a security issue +# If we need to bootstrap a MDS we should probably create the following on the monitors +# I understand that this handy to do this here +# but having the admin key inside every container is a concern + +# Create the Ceph filesystem, if necessary +if [ $CEPHFS_CREATE -eq 1 ]; then + + if [[ ! -e ${ADMIN_KEYRING} ]]; then + echo "ERROR- ${ADMIN_KEYRING} must exist; get it from your existing mon" + exit 1 + fi + + if [[ "$(ceph --cluster "${CLUSTER}" fs ls | grep -c name:.${CEPHFS_NAME},)" -eq 0 ]]; then + # Make sure the specified data pool exists + if ! ceph --cluster "${CLUSTER}" osd pool stats ${CEPHFS_DATA_POOL} > /dev/null 2>&1; then + ceph --cluster "${CLUSTER}" osd pool create ${CEPHFS_DATA_POOL} ${CEPHFS_DATA_POOL_PG} + fi + + # Make sure the specified metadata pool exists + if ! ceph --cluster "${CLUSTER}" osd pool stats ${CEPHFS_METADATA_POOL} > /dev/null 2>&1; then + ceph --cluster "${CLUSTER}" osd pool create ${CEPHFS_METADATA_POOL} ${CEPHFS_METADATA_POOL_PG} + fi + + ceph --cluster "${CLUSTER}" fs new ${CEPHFS_NAME} ${CEPHFS_METADATA_POOL} ${CEPHFS_DATA_POOL} + fi +fi + +# NOTE: prefixing this with exec causes it to die (commit suicide) +/usr/bin/ceph-mds \ + --cluster "${CLUSTER}" \ + --setuser "ceph" \ + --setgroup "ceph" \ + -d \ + -i "${MDS_NAME}" diff --git a/ceph-client/templates/bin/pool/_calc.py.tpl b/ceph-client/templates/bin/pool/_calc.py.tpl new file mode 100644 index 0000000000..4409a52847 --- /dev/null +++ b/ceph-client/templates/bin/pool/_calc.py.tpl @@ -0,0 +1,44 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +#NOTE(portdirect): this is a simple approximation of https://ceph.com/pgcalc/ + +import math +import sys + +replication = int(sys.argv[1]) +number_of_osds = int(sys.argv[2]) +percentage_data = float(sys.argv[3]) +target_pgs_per_osd = int(sys.argv[4]) + +raw_pg_num_opt = target_pgs_per_osd * number_of_osds \ + * (math.ceil(percentage_data) / 100.0) / replication + +raw_pg_num_min = number_of_osds / replication + +if raw_pg_num_min >= raw_pg_num_opt: + raw_pg_num = raw_pg_num_min +else: + raw_pg_num = raw_pg_num_opt + +max_pg_num = int(math.pow(2, math.ceil(math.log(raw_pg_num, 2)))) +min_pg_num = int(math.pow(2, math.floor(math.log(raw_pg_num, 2)))) + +if min_pg_num >= (raw_pg_num * 0.75): + print(min_pg_num) +else: + print(max_pg_num) diff --git a/ceph-client/templates/bin/pool/_init.sh.tpl b/ceph-client/templates/bin/pool/_init.sh.tpl new file mode 100644 index 0000000000..07ac4726e2 --- /dev/null +++ b/ceph-client/templates/bin/pool/_init.sh.tpl @@ -0,0 +1,460 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +export LC_ALL=C + +: "${ADMIN_KEYRING:=/etc/ceph/${CLUSTER}.client.admin.keyring}" +: "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}" + +{{ include "helm-toolkit.snippets.mon_host_from_k8s_ep" . }} + +if [[ ! -e ${CEPH_CONF}.template ]]; then + echo "ERROR- ${CEPH_CONF}.template must exist; get it from your existing mon" + exit 1 +else + ENDPOINT=$(mon_host_from_k8s_ep "${NAMESPACE}" ceph-mon-discovery) + if [[ "${ENDPOINT}" == "" ]]; then + /bin/sh -c -e "cat ${CEPH_CONF}.template | tee ${CEPH_CONF}" || true + else + /bin/sh -c -e "cat ${CEPH_CONF}.template | sed 's#mon_host.*#mon_host = ${ENDPOINT}#g' | tee ${CEPH_CONF}" || true + fi +fi + +if [[ ! -e ${ADMIN_KEYRING} ]]; then + echo "ERROR- ${ADMIN_KEYRING} must exist; get it from your existing mon" + exit 1 +fi + +function wait_for_pid() { + tail --pid=$1 -f /dev/null +} + +function wait_for_pgs () { + echo "#### Start: Checking pgs ####" + + pgs_ready=0 + query='map({state: .state}) | group_by(.state) | map({state: .[0].state, count: length}) | .[] | select(.state | contains("active") or contains("premerge") | not)' + + if [[ $(ceph mon versions | awk '/version/{print $3}' | sort -n | head -n 1 | cut -d. -f1) -ge 14 ]]; then + query=".pg_stats | ${query}" + fi + + # Loop until all pgs are active + while [[ $pgs_ready -lt 3 ]]; do + pgs_state=$(ceph --cluster ${CLUSTER} pg ls -f json | jq -c "${query}") + if [[ $(jq -c '. | select(.state | contains("peer") or contains("activating") or contains("recover") or contains("unknown") or contains("creating") | not)' <<< "${pgs_state}") ]]; then + # If inactive PGs aren't in the allowed set of states above, fail + echo "Failure, found inactive PGs that aren't in the allowed set of states" + exit 1 + fi + if [[ "${pgs_state}" ]]; then + pgs_ready=0 + else + (( pgs_ready+=1 )) + fi + sleep 3 + done +} + +function check_recovery_flags () { + echo "### Start: Checking for flags that will prevent recovery" + + # Ensure there are no flags set that will prevent recovery of degraded PGs + if [[ $(ceph osd stat | grep "norecover\|nobackfill\|norebalance") ]]; then + ceph osd stat + echo "Flags are set that prevent recovery of degraded PGs" + exit 1 + fi +} + +function check_osd_count() { + echo "#### Start: Checking OSD count ####" + noup_flag=$(ceph osd stat | awk '/noup/ {print $2}') + osd_stat=$(ceph osd stat -f json-pretty) + num_osd=$(awk '/"num_osds"/{print $2}' <<< "$osd_stat" | cut -d, -f1) + num_in_osds=$(awk '/"num_in_osds"/{print $2}' <<< "$osd_stat" | cut -d, -f1) + num_up_osds=$(awk '/"num_up_osds"/{print $2}' <<< "$osd_stat" | cut -d, -f1) + + EXPECTED_OSDS={{.Values.conf.pool.target.osd}} + EXPECTED_FINAL_OSDS={{.Values.conf.pool.target.final_osd}} + REQUIRED_PERCENT_OF_OSDS={{.Values.conf.pool.target.required_percent_of_osds}} + + if [ ${num_up_osds} -gt ${EXPECTED_FINAL_OSDS} ]; then + echo "More running OSDs (${num_up_osds}) than expected (${EXPECTED_FINAL_OSDS}). Please correct the expected value (.Values.conf.pool.target.final_osd)." + exit 1 + fi + + MIN_OSDS=$(($EXPECTED_OSDS*$REQUIRED_PERCENT_OF_OSDS/100)) + if [ ${MIN_OSDS} -lt 1 ]; then + MIN_OSDS=1 + fi + + if [ "${noup_flag}" ]; then + osd_status=$(ceph osd dump -f json | jq -c '.osds[] | .state') + count=0 + for osd in $osd_status; do + if [[ "$osd" == *"up"* || "$osd" == *"new"* ]]; then + ((count=count+1)) + fi + done + echo "Caution: noup flag is set. ${count} OSDs in up/new state. Required number of OSDs: ${MIN_OSDS}." + if [ $MIN_OSDS -gt $count ]; then + exit 1 + fi + else + if [ "${num_osd}" -eq 0 ]; then + echo "There are no osds in the cluster" + exit 1 + elif [ "${num_in_osds}" -ge "${MIN_OSDS}" ] && [ "${num_up_osds}" -ge "${MIN_OSDS}" ]; then + echo "Required number of OSDs (${MIN_OSDS}) are UP and IN status" + else + echo "Required number of OSDs (${MIN_OSDS}) are NOT UP and IN status. Cluster shows OSD count=${num_osd}, UP=${num_up_osds}, IN=${num_in_osds}" + exit 1 + fi + fi +} + +function create_crushrule () { + CRUSH_NAME=$1 + CRUSH_RULE=$2 + CRUSH_FAILURE_DOMAIN=$3 + CRUSH_DEVICE_CLASS=$4 + if ! ceph --cluster "${CLUSTER}" osd crush rule ls | grep -q "^\$CRUSH_NAME$"; then + ceph --cluster "${CLUSTER}" osd crush rule $CRUSH_RULE $CRUSH_NAME default $CRUSH_FAILURE_DOMAIN $CRUSH_DEVICE_CLASS || true + fi +} + +# Set mons to use the msgr2 protocol on nautilus +if [[ $(ceph mon versions | awk '/version/{print $3}' | cut -d. -f1) -ge 14 ]]; then + ceph --cluster "${CLUSTER}" mon enable-msgr2 +fi + +check_osd_count +{{- range $crush_rule := .Values.conf.pool.crush_rules -}} +{{- with $crush_rule }} +create_crushrule {{ .name }} {{ .crush_rule }} {{ .failure_domain }} {{ .device_class }} +{{- end }} +{{- end }} + +function reweight_osds () { + OSD_DF_OUTPUT=$(ceph --cluster "${CLUSTER}" osd df --format json-pretty) + for OSD_ID in $(ceph --cluster "${CLUSTER}" osd ls); do + OSD_EXPECTED_WEIGHT=$(echo "${OSD_DF_OUTPUT}" | grep -A7 "\bosd.${OSD_ID}\b" | awk '/"kb"/{ gsub(",",""); d= $2/1073741824 ; r = sprintf("%.2f", d); print r }'); + OSD_WEIGHT=$(echo "${OSD_DF_OUTPUT}" | grep -A3 "\bosd.${OSD_ID}\b" | awk '/crush_weight/{print $2}' | cut -d',' -f1) + if [[ "${OSD_EXPECTED_WEIGHT}" != "0.00" ]] && [[ "${OSD_WEIGHT}" != "${OSD_EXPECTED_WEIGHT}" ]]; then + ceph --cluster "${CLUSTER}" osd crush reweight osd.${OSD_ID} ${OSD_EXPECTED_WEIGHT}; + fi + done +} + +function enable_autoscaling () { + CEPH_MAJOR_VERSION=$(ceph mgr versions | awk '/version/{print $3}' | cut -d. -f1) + + if [[ ${CEPH_MAJOR_VERSION} -ge 16 ]]; then + # Pacific introduced the noautoscale flag to make this simpler + ceph osd pool unset noautoscale + else + if [[ ${CEPH_MAJOR_VERSION} -eq 14 ]]; then + ceph mgr module enable pg_autoscaler # only required for nautilus + fi + ceph config set global osd_pool_default_pg_autoscale_mode on + fi +} + +function disable_autoscaling () { + CEPH_MAJOR_VERSION=$(ceph mgr versions | awk '/version/{print $3}' | cut -d. -f1) + + if [[ ${CEPH_MAJOR_VERSION} -ge 16 ]]; then + # Pacific introduced the noautoscale flag to make this simpler + ceph osd pool set noautoscale + else + if [[ ${CEPH_MAJOR_VERSION} -eq 14 ]]; then + ceph mgr module disable pg_autoscaler # only required for nautilus + fi + ceph config set global osd_pool_default_pg_autoscale_mode off + fi +} + +function set_cluster_flags () { + if [[ -n "${CLUSTER_SET_FLAGS}" ]]; then + for flag in ${CLUSTER_SET_FLAGS}; do + ceph osd set ${flag} + done + fi +} + +function unset_cluster_flags () { + if [[ -n "${CLUSTER_UNSET_FLAGS}" ]]; then + for flag in ${CLUSTER_UNSET_FLAGS}; do + ceph osd unset ${flag} + done + fi +} + +function run_cluster_commands () { + {{- range .Values.conf.features.cluster_commands }} + ceph --cluster "${CLUSTER}" {{ . }} + {{- end }} +} + +# Helper function to set pool properties only if the target value differs from +# the current value to optimize performance +function set_pool_property() { + POOL_NAME=$1 + PROPERTY_NAME=$2 + CURRENT_PROPERTY_VALUE=$3 + TARGET_PROPERTY_VALUE=$4 + REALLY_MEAN_IT="" + + if [[ "${PROPERTY_NAME}" == "size" ]]; then + REALLY_MEAN_IT="--yes-i-really-mean-it" + fi + + if [[ "${CURRENT_PROPERTY_VALUE}" != "${TARGET_PROPERTY_VALUE}" ]]; then + ceph --cluster "${CLUSTER}" osd pool set "${POOL_NAME}" "${PROPERTY_NAME}" "${TARGET_PROPERTY_VALUE}" ${REALLY_MEAN_IT} + fi + + echo "${TARGET_PROPERTY_VALUE}" +} + +function create_pool () { + POOL_APPLICATION=$1 + POOL_NAME=$2 + POOL_REPLICATION=$3 + POOL_PLACEMENT_GROUPS=$4 + POOL_CRUSH_RULE=$5 + POOL_PROTECTION=$6 + PG_NUM_MIN=$7 + if ! ceph --cluster "${CLUSTER}" osd pool stats "${POOL_NAME}" > /dev/null 2>&1; then + if [[ ${POOL_PLACEMENT_GROUPS} -gt 0 ]]; then + ceph --cluster "${CLUSTER}" osd pool create "${POOL_NAME}" ${POOL_PLACEMENT_GROUPS} + else + ceph --cluster "${CLUSTER}" osd pool create "${POOL_NAME}" ${PG_NUM_MIN} --pg-num-min ${PG_NUM_MIN} + fi + while [ $(ceph --cluster "${CLUSTER}" -s | grep creating -c) -gt 0 ]; do echo -n .;sleep 1; done + ceph --cluster "${CLUSTER}" osd pool application enable "${POOL_NAME}" "${POOL_APPLICATION}" + fi + + # 'tr' and 'awk' are needed here to strip off text that is echoed before the JSON string. + # In some cases, errors/warnings are written to stdout and the JSON doesn't parse correctly. + pool_values=$(ceph --cluster "${CLUSTER}" osd pool get "${POOL_NAME}" all -f json | tr -d '\n' | awk -F{ '{print "{" $2}') + + if [[ $(ceph mgr versions | awk '/version/{print $3}' | cut -d. -f1) -ge 14 ]]; then + if [[ "${ENABLE_AUTOSCALER}" == "true" ]]; then + pg_num=$(jq -r '.pg_num' <<< "${pool_values}") + pgp_num=$(jq -r '.pgp_num' <<< "${pool_values}") + pg_num_min=$(jq -r '.pg_num_min' <<< "${pool_values}") + pg_autoscale_mode=$(jq -r '.pg_autoscale_mode' <<< "${pool_values}") + # set pg_num_min to PG_NUM_MIN before enabling autoscaler + if [[ ${pg_num} -lt ${PG_NUM_MIN} ]]; then + pg_autoscale_mode=$(set_pool_property "${POOL_NAME}" pg_autoscale_mode "${pg_autoscale_mode}" "off") + pg_num=$(set_pool_property "${POOL_NAME}" pg_num "${pg_num}" "${PG_NUM_MIN}") + pgp_num=$(set_pool_property "${POOL_NAME}" pgp_num "${pgp_num}" "${PG_NUM_MIN}") + fi + pg_num_min=$(set_pool_property "${POOL_NAME}" pg_num_min "${pg_num_min}" "${PG_NUM_MIN}") + pg_autoscale_mode=$(set_pool_property "${POOL_NAME}" pg_autoscale_mode "${pg_autoscale_mode}" "on") + else + pg_autoscale_mode=$(set_pool_property "${POOL_NAME}" pg_autoscale_mode "${pg_autoscale_mode}" "off") + fi + fi +# +# Make sure pool is not protected after creation AND expansion so we can manipulate its settings. +# Final protection settings are applied once parameters (size, pg) have been adjusted. +# + nosizechange=$(jq -r '.nosizechange' <<< "${pool_values}") + nopschange=$(jq -r '.nopschange' <<< "${pool_values}") + nodelete=$(jq -r '.nodelete' <<< "${pool_values}") + size=$(jq -r '.size' <<< "${pool_values}") + crush_rule=$(jq -r '.crush_rule' <<< "${pool_values}") + nosizechange=$(set_pool_property "${POOL_NAME}" nosizechange "${nosizechange}" "false") + nopgchange=$(set_pool_property "${POOL_NAME}" nopgchange "${nopgchange}" "false") + nodelete=$(set_pool_property "${POOL_NAME}" nodelete "${nodelete}" "false") + size=$(set_pool_property "${POOL_NAME}" size "${size}" "${POOL_REPLICATION}") + crush_rule=$(set_pool_property "${POOL_NAME}" crush_rule "${crush_rule}" "${POOL_CRUSH_RULE}") +# set pg_num to pool + if [[ ${POOL_PLACEMENT_GROUPS} -gt 0 ]]; then + pg_num=$(jq -r ".pg_num" <<< "${pool_values}") + pgp_num=$(jq -r ".pgp_num" <<< "${pool_values}") + pg_num=$(set_pool_property "${POOL_NAME}" pg_num "${pg_num}" "${POOL_PLACEMENT_GROUPS}") + pgp_num=$(set_pool_property "${POOL_NAME}" pgp_num "${pgp_num}" "${POOL_PLACEMENT_GROUPS}") + fi + +#This is to handle cluster expansion case where replication may change from intilization + if [ ${POOL_REPLICATION} -gt 1 ]; then + min_size=$(jq -r '.min_size' <<< "${pool_values}") + EXPECTED_POOLMINSIZE=$[${POOL_REPLICATION}-1] + min_size=$(set_pool_property "${POOL_NAME}" min_size "${min_size}" "${EXPECTED_POOLMINSIZE}") + fi +# +# Handling of .Values.conf.pool.target.protected: +# Possible settings +# - true | 1 = Protect the pools after they get created +# - false | 0 = Do not protect the pools once they get created and let Ceph defaults apply +# - Absent = Do not protect the pools once they get created and let Ceph defaults apply +# +# If protection is not requested through values.yaml, just use the Ceph defaults. With Luminous we do not +# apply any protection to the pools when they get created. +# +# Note: If the /etc/ceph/ceph.conf file modifies the defaults the deployment will fail on pool creation +# - nosizechange = Do not allow size and min_size changes on the pool +# - nodelete = Do not allow deletion of the pool +# + if [ "x${POOL_PROTECTION}" == "xtrue" ] || [ "x${POOL_PROTECTION}" == "x1" ]; then + nosizechange=$(set_pool_property "${POOL_NAME}" nosizechange "${nosizechange}" "true") + nodelete=$(set_pool_property "${POOL_NAME}" nodelete "${nodelete}" "true") + fi +} + +function manage_pool () { + POOL_APPLICATION=$1 + POOL_NAME=$2 + POOL_REPLICATION=$3 + TOTAL_DATA_PERCENT=$4 + TARGET_PG_PER_OSD=$5 + POOL_CRUSH_RULE=$6 + POOL_QUOTA=$7 + POOL_PROTECTION=$8 + CLUSTER_CAPACITY=$9 + POOL_PG_NUM_MIN=${10} + TOTAL_OSDS={{.Values.conf.pool.target.osd}} + POOL_PLACEMENT_GROUPS=0 + if [[ -n "${TOTAL_DATA_PERCENT}" ]]; then + if [[ "${ENABLE_AUTOSCALER}" == "false" ]] || [[ $(ceph mgr versions | awk '/version/{print $3}' | cut -d. -f1) -lt 14 ]]; then + POOL_PLACEMENT_GROUPS=$(python3 /tmp/pool-calc.py ${POOL_REPLICATION} ${TOTAL_OSDS} ${TOTAL_DATA_PERCENT} ${TARGET_PG_PER_OSD}) + fi + fi + create_pool "${POOL_APPLICATION}" "${POOL_NAME}" "${POOL_REPLICATION}" "${POOL_PLACEMENT_GROUPS}" "${POOL_CRUSH_RULE}" "${POOL_PROTECTION}" "${POOL_PG_NUM_MIN}" + ceph --cluster "${CLUSTER}" osd pool set-quota "${POOL_NAME}" max_bytes $POOL_QUOTA +} + +# Helper to convert TiB, TB, GiB, GB, MiB, MB, KiB, KB, or bytes to bytes +function convert_to_bytes() { + value=${1} + value="$(echo "${value}" | sed 's/TiB/ \* 1024GiB/g')" + value="$(echo "${value}" | sed 's/TB/ \* 1000GB/g')" + value="$(echo "${value}" | sed 's/GiB/ \* 1024MiB/g')" + value="$(echo "${value}" | sed 's/GB/ \* 1000MB/g')" + value="$(echo "${value}" | sed 's/MiB/ \* 1024KiB/g')" + value="$(echo "${value}" | sed 's/MB/ \* 1000KB/g')" + value="$(echo "${value}" | sed 's/KiB/ \* 1024/g')" + value="$(echo "${value}" | sed 's/KB/ \* 1000/g')" + python3 -c "print(int(${value}))" +} + +set_cluster_flags +unset_cluster_flags +run_cluster_commands +reweight_osds + +{{ $targetOSDCount := .Values.conf.pool.target.osd }} +{{ $targetFinalOSDCount := .Values.conf.pool.target.final_osd }} +{{ $targetPGperOSD := .Values.conf.pool.target.pg_per_osd }} +{{ $crushRuleDefault := .Values.conf.pool.default.crush_rule }} +{{ $targetQuota := .Values.conf.pool.target.quota | default 100 }} +{{ $targetProtection := .Values.conf.pool.target.protected | default "false" | quote | lower }} +{{ $targetPGNumMin := .Values.conf.pool.target.pg_num_min }} +cluster_capacity=$(ceph --cluster "${CLUSTER}" df -f json-pretty | grep '"total_bytes":' | head -n1 | awk '{print $2}' | tr -d ',') + +# Check to make sure pool quotas don't exceed the expected cluster capacity in its final state +target_quota=$(python3 -c "print(int(${cluster_capacity} * {{ $targetFinalOSDCount }} / {{ $targetOSDCount }} * {{ $targetQuota }} / 100))") +quota_sum=0 + +{{- range $pool := .Values.conf.pool.spec -}} +{{- with $pool }} +# Read the pool quota from the pool spec (no quota if absent) +# Set pool_quota to 0 if target_quota is 0 +[[ ${target_quota} -eq 0 ]] && pool_quota=0 || pool_quota="$(convert_to_bytes {{ .pool_quota | default 0 }})" +quota_sum=$(python3 -c "print(int(${quota_sum} + (${pool_quota} * {{ .replication }})))") +{{- end }} +{{- end }} + +if [[ ${quota_sum} -gt ${target_quota} ]]; then + echo "The sum of all pool quotas exceeds the target quota for the cluster" + exit 1 +fi + +if [[ $(ceph mgr versions | awk '/version/{print $3}' | cut -d. -f1) -ge 14 ]] && [[ "${ENABLE_AUTOSCALER}" != "true" ]]; then + disable_autoscaling +fi + +# Track the manage_pool() PIDs in an array so we can wait for them to finish +MANAGE_POOL_PIDS=() + +{{- range $pool := .Values.conf.pool.spec -}} +{{- with $pool }} +pool_name="{{ .name }}" +{{- if .rename }} +# If a renamed pool exists, that name should be used for idempotence +if [[ -n "$(ceph --cluster ${CLUSTER} osd pool ls | grep ^{{ .rename }}$)" ]]; then + pool_name="{{ .rename }}" +fi +{{- end }} +# Read the pool quota from the pool spec (no quota if absent) +# Set pool_quota to 0 if target_quota is 0 +[[ ${target_quota} -eq 0 ]] && pool_quota=0 || pool_quota="$(convert_to_bytes {{ .pool_quota | default 0 }})" +pool_crush_rule="{{ $crushRuleDefault }}" +{{- if .crush_rule }} +pool_crush_rule="{{ .crush_rule }}" +{{- end }} +pool_pg_num_min={{ $targetPGNumMin }} +{{- if .pg_num_min }} +pool_pg_num_min={{ .pg_num_min }} +{{- end }} +manage_pool {{ .application }} ${pool_name} {{ .replication }} {{ .percent_total_data }} {{ $targetPGperOSD }} $pool_crush_rule $pool_quota {{ $targetProtection }} ${cluster_capacity} ${pool_pg_num_min} & +MANAGE_POOL_PID=$! +MANAGE_POOL_PIDS+=( $MANAGE_POOL_PID ) +{{- if .rename }} +# Wait for manage_pool() to finish for this pool before trying to rename the pool +wait_for_pid $MANAGE_POOL_PID +# If a rename value exists, the pool exists, and a pool with the rename value doesn't exist, rename the pool +pool_list=$(ceph --cluster ${CLUSTER} osd pool ls) +if [[ -n $(grep ^{{ .name }}$ <<< "${pool_list}") ]] && + [[ -z $(grep ^{{ .rename }}$ <<< "${pool_list}") ]]; then + ceph --cluster "${CLUSTER}" osd pool rename "{{ .name }}" "{{ .rename }}" + pool_name="{{ .rename }}" +fi +{{- end }} +{{- if and .delete .delete_all_pool_data }} +# Wait for manage_pool() to finish for this pool before trying to delete the pool +wait_for_pid $MANAGE_POOL_PID +# If delete is set to true and delete_all_pool_data is also true, delete the pool +if [[ "true" == "{{ .delete }}" ]] && + [[ "true" == "{{ .delete_all_pool_data }}" ]]; then + ceph --cluster "${CLUSTER}" tell mon.* injectargs '--mon-allow-pool-delete=true' + ceph --cluster "${CLUSTER}" osd pool set "${pool_name}" nodelete false + ceph --cluster "${CLUSTER}" osd pool delete "${pool_name}" "${pool_name}" --yes-i-really-really-mean-it + ceph --cluster "${CLUSTER}" tell mon.* injectargs '--mon-allow-pool-delete=false' +fi +{{- end }} +{{- end }} +{{- end }} + +# Wait for all manage_pool() instances to finish before proceeding +for pool_pid in ${MANAGE_POOL_PIDS[@]}; do + wait_for_pid $pool_pid +done + +if [[ $(ceph mgr versions | awk '/version/{print $3}' | cut -d. -f1) -ge 14 ]] && [[ "${ENABLE_AUTOSCALER}" == "true" ]]; then + enable_autoscaling +fi + +{{- if .Values.conf.pool.crush.tunables }} +ceph --cluster "${CLUSTER}" osd crush tunables {{ .Values.conf.pool.crush.tunables }} +{{- end }} + +wait_for_pgs +check_recovery_flags diff --git a/ceph-client/templates/bin/utils/_checkDNS.sh.tpl b/ceph-client/templates/bin/utils/_checkDNS.sh.tpl new file mode 100644 index 0000000000..b7e360b2fe --- /dev/null +++ b/ceph-client/templates/bin/utils/_checkDNS.sh.tpl @@ -0,0 +1,38 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +: "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}" +ENDPOINT="{$1}" + +function check_mon_dns () { + GREP_CMD=$(grep -rl 'ceph-mon' ${CEPH_CONF}) + + if [[ "${ENDPOINT}" == "{up}" ]]; then + echo "If DNS is working, we are good here" + elif [[ "${ENDPOINT}" != "" ]]; then + if [[ ${GREP_CMD} != "" ]]; then + # No DNS, write CEPH MONs IPs into ${CEPH_CONF} + sh -c -e "cat ${CEPH_CONF}.template | sed 's/mon_host.*/mon_host = ${ENDPOINT}/g' | tee ${CEPH_CONF}" > /dev/null 2>&1 + else + echo "endpoints are already cached in ${CEPH_CONF}" + exit + fi + fi +} + +check_mon_dns + +exit diff --git a/ceph-client/templates/bin/utils/_checkDNS_start.sh.tpl b/ceph-client/templates/bin/utils/_checkDNS_start.sh.tpl new file mode 100644 index 0000000000..b4167200f9 --- /dev/null +++ b/ceph-client/templates/bin/utils/_checkDNS_start.sh.tpl @@ -0,0 +1,70 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -xe + +{{ include "helm-toolkit.snippets.mon_host_from_k8s_ep" . }} + +{{- $rgwNameSpaces := "" }} +{{- $sep := "" }} +{{- range $_, $ns := .Values.endpoints.ceph_object_store.endpoint_namespaces }} + {{- $rgwNameSpaces = printf "%s%s%s" $rgwNameSpaces $sep $ns }} + {{- $sep = " " }} +{{- end }} + +rgwNameSpaces={{- printf "\"%s\"" $rgwNameSpaces }} + +function check_mon_dns { + NS=${1} + # RGWs and the rgw namespace could not exist. Let's check this and prevent this script from failing + if [[ $(kubectl get ns ${NS} -o json | jq -r '.status.phase') == "Active" ]]; then + DNS_CHECK=$(getent hosts ceph-mon | head -n1) + PODS=$(kubectl get pods --namespace=${NS} --selector=application=ceph --field-selector=status.phase=Running \ + --output=jsonpath='{range .items[*]}{.metadata.name}{"\n"}{end}' | grep -E 'ceph-mon|ceph-osd|ceph-mgr|ceph-mds|ceph-rgw') + ENDPOINT=$(mon_host_from_k8s_ep "${NAMESPACE}" ceph-mon-discovery) + + if [[ ${PODS} == "" || "${ENDPOINT}" == "" ]]; then + echo "Something went wrong, no PODS or ENDPOINTS are available!" + elif [[ ${DNS_CHECK} == "" ]]; then + for POD in ${PODS}; do + kubectl exec -t ${POD} --namespace=${NS} -- \ + sh -c -e "/tmp/utils-checkDNS.sh "${ENDPOINT}"" + done + else + for POD in ${PODS}; do + kubectl exec -t ${POD} --namespace=${NS} -- \ + sh -c -e "/tmp/utils-checkDNS.sh up" + done + fi + else + echo "The namespace ${NS} is not ready, yet" + fi +} + +function watch_mon_dns { + while [ true ]; do + echo "checking DNS health" + for myNS in ${NAMESPACE} ${rgwNameSpaces}; do + check_mon_dns ${myNS} || true + done + echo "sleep 300 sec" + sleep 300 + done +} + +watch_mon_dns + +exit diff --git a/ceph-client/templates/bin/utils/_checkPGs.sh.tpl b/ceph-client/templates/bin/utils/_checkPGs.sh.tpl new file mode 100644 index 0000000000..1a820ca2f5 --- /dev/null +++ b/ceph-client/templates/bin/utils/_checkPGs.sh.tpl @@ -0,0 +1,21 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +mgrPod=$(kubectl get pods --namespace=${DEPLOYMENT_NAMESPACE} --selector=application=ceph --selector=component=mgr --output=jsonpath={.items[0].metadata.name} 2>/dev/null) + +kubectl exec -t ${mgrPod} --namespace=${DEPLOYMENT_NAMESPACE} -- python3 /tmp/utils-checkPGs.py All 2>/dev/null diff --git a/ceph-client/templates/bin/utils/_defragOSDs.sh.tpl b/ceph-client/templates/bin/utils/_defragOSDs.sh.tpl new file mode 100644 index 0000000000..68d0946643 --- /dev/null +++ b/ceph-client/templates/bin/utils/_defragOSDs.sh.tpl @@ -0,0 +1,29 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +PODS=$(kubectl get pods --namespace=${NAMESPACE} \ + --selector=application=ceph,component=osd --field-selector=status.phase=Running \ + '--output=jsonpath={range .items[*]}{.metadata.name}{"\n"}{end}') + +for POD in ${PODS}; do + kubectl exec -t ${POD} -c ceph-osd-default --namespace=${NAMESPACE} -- \ + sh -c -e "/tmp/utils-defragOSDs.sh" +done + + +exit 0 diff --git a/ceph-client/templates/configmap-bin.yaml b/ceph-client/templates/configmap-bin.yaml new file mode 100644 index 0000000000..04a9987ffd --- /dev/null +++ b/ceph-client/templates/configmap-bin.yaml @@ -0,0 +1,57 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.configmap_bin .Values.deployment.ceph }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ceph-client-bin +data: +{{- if .Values.images.local_registry.active }} + image-repo-sync.sh: | +{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} +{{- end }} + +{{- if .Values.bootstrap.enabled }} + bootstrap.sh: | +{{ tuple "bin/_bootstrap.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} + + init-dirs.sh: | +{{ tuple "bin/_init-dirs.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + + pool-init.sh: | +{{ tuple "bin/pool/_init.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + pool-calc.py: | +{{ tuple "bin/pool/_calc.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + + mds-start.sh: | +{{ tuple "bin/mds/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + + helm-tests.sh: | +{{ tuple "bin/_helm-tests.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + utils-checkDNS.sh: | +{{ tuple "bin/utils/_checkDNS.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + utils-checkDNS_start.sh: | +{{ tuple "bin/utils/_checkDNS_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + + utils-checkPGs.sh: | +{{ tuple "bin/utils/_checkPGs.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + + utils-defragOSDs.sh: | +{{ tuple "bin/utils/_defragOSDs.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + +{{- end }} diff --git a/ceph-client/templates/configmap-etc-client.yaml b/ceph-client/templates/configmap-etc-client.yaml new file mode 100644 index 0000000000..c849b79af0 --- /dev/null +++ b/ceph-client/templates/configmap-etc-client.yaml @@ -0,0 +1,50 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "ceph.configmap.etc" }} +{{- $configMapName := index . 0 }} +{{- $envAll := index . 1 }} +{{- with $envAll }} + +{{- if .Values.deployment.ceph }} + +{{- if empty .Values.conf.ceph.global.mon_host -}} +{{- $monHost := tuple "ceph_mon" "internal" "mon_msgr2" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} +{{- $_ := $monHost | set .Values.conf.ceph.global "mon_host" -}} +{{- end -}} + + +{{- if empty .Values.conf.ceph.osd.cluster_network -}} +{{- $_ := .Values.network.cluster | set .Values.conf.ceph.osd "cluster_network" -}} +{{- end -}} + +{{- if empty .Values.conf.ceph.osd.public_network -}} +{{- $_ := .Values.network.public | set .Values.conf.ceph.osd "public_network" -}} +{{- end -}} + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $configMapName }} +data: + ceph.conf: | +{{ include "helm-toolkit.utils.to_ini" .Values.conf.ceph | indent 4 }} + +{{- end }} +{{- end }} +{{- end }} +{{- if .Values.manifests.configmap_etc }} +{{- list "ceph-client-etc" . | include "ceph.configmap.etc" }} +{{- end }} diff --git a/ceph-client/templates/cronjob-checkPGs.yaml b/ceph-client/templates/cronjob-checkPGs.yaml new file mode 100644 index 0000000000..9f96518e9c --- /dev/null +++ b/ceph-client/templates/cronjob-checkPGs.yaml @@ -0,0 +1,150 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.cronjob_checkPGs }} +{{- $envAll := . }} + +{{- $serviceAccountName := "ceph-pool-checkpgs" }} +{{ tuple $envAll "pool_checkpgs" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $serviceAccountName }} +rules: + - apiGroups: + - "" + resources: + - pods + - pods/exec + verbs: + - get + - list + - watch + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $serviceAccountName }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ $serviceAccountName }} + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "ceph" "pool-checkpgs" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + schedule: {{ .Values.jobs.pool_checkPGs.cron | quote }} + successfulJobsHistoryLimit: {{ .Values.jobs.pool_checkPGs.history.successJob }} + failedJobsHistoryLimit: {{ .Values.jobs.pool_checkPGs.history.failJob }} + concurrencyPolicy: {{ .Values.jobs.pool_checkPGs.concurrency.execPolicy }} + startingDeadlineSeconds: {{ .Values.jobs.pool_checkPGs.startingDeadlineSecs }} + jobTemplate: + metadata: + labels: +{{ tuple $envAll "ceph" "pool-checkpgs" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + spec: + template: + metadata: + labels: +{{ tuple $envAll "ceph" "pool-checkpgs" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 12 }} + spec: + serviceAccountName: {{ $serviceAccountName }} + nodeSelector: + {{ .Values.labels.mgr.node_selector_key }}: {{ .Values.labels.mgr.node_selector_value }} + initContainers: +{{ tuple $envAll "pool_checkpgs" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 12 }} + containers: + - name: {{ $serviceAccountName }} +{{ tuple $envAll "ceph_config_helper" | include "helm-toolkit.snippets.image" | indent 12 }} + env: + - name: DEPLOYMENT_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + command: + - /tmp/utils-checkPGs.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: pod-etc-ceph + mountPath: /etc/ceph + - name: ceph-client-bin + mountPath: /tmp/utils-checkPGs.sh + subPath: utils-checkPGs.sh + readOnly: true + - name: ceph-client-etc + mountPath: /etc/ceph/ceph.conf + subPath: ceph.conf + readOnly: true + - mountPath: /etc/ceph/ceph.client.admin.keyring + name: ceph-client-admin-keyring + readOnly: true + subPath: ceph.client.admin.keyring + - mountPath: /etc/ceph/ceph.mon.keyring.seed + name: ceph-mon-keyring + readOnly: true + subPath: ceph.mon.keyring + - mountPath: /var/lib/ceph/bootstrap-osd/ceph.keyring + name: ceph-bootstrap-osd-keyring + readOnly: true + subPath: ceph.keyring + - mountPath: /var/lib/ceph/bootstrap-mds/ceph.keyring + name: ceph-bootstrap-mds-keyring + readOnly: true + subPath: ceph.keyring + restartPolicy: Never + hostNetwork: true + volumes: + - name: pod-tmp + emptyDir: {} + - name: pod-etc-ceph + emptyDir: {} + - name: ceph-client-bin + configMap: + name: ceph-client-bin + defaultMode: 0555 + - name: ceph-client-etc + configMap: + name: ceph-client-etc + defaultMode: 0444 + - name: ceph-client-admin-keyring + secret: + defaultMode: 420 + secretName: ceph-client-admin-keyring + - name: ceph-mon-keyring + secret: + defaultMode: 420 + secretName: ceph-mon-keyring + - name: ceph-bootstrap-osd-keyring + secret: + defaultMode: 420 + secretName: ceph-bootstrap-osd-keyring + - name: ceph-bootstrap-mds-keyring + secret: + defaultMode: 420 + secretName: ceph-bootstrap-mds-keyring + +{{- end }} diff --git a/ceph-client/templates/cronjob-defragosds.yaml b/ceph-client/templates/cronjob-defragosds.yaml new file mode 100644 index 0000000000..38fc5b6802 --- /dev/null +++ b/ceph-client/templates/cronjob-defragosds.yaml @@ -0,0 +1,110 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.cronjob_defragosds }} +{{- $envAll := . }} + +{{- $serviceAccountName := "ceph-defragosds" }} +{{ tuple $envAll "ceph_defragosds" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $serviceAccountName }} +rules: + - apiGroups: + - "" + resources: + - pods + - pods/exec + verbs: + - get + - list + - watch + - create +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $serviceAccountName }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: {{ $serviceAccountName }} + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "ceph" "ceph-defragosds" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + schedule: {{ .Values.jobs.ceph_defragosds.cron | quote }} + successfulJobsHistoryLimit: {{ .Values.jobs.ceph_defragosds.history.successJob }} + failedJobsHistoryLimit: {{ .Values.jobs.ceph_defragosds.history.failJob }} + concurrencyPolicy: {{ .Values.jobs.ceph_defragosds.concurrency.execPolicy }} + startingDeadlineSeconds: {{ .Values.jobs.ceph_defragosds.startingDeadlineSecs }} + jobTemplate: + metadata: + labels: +{{ tuple $envAll "ceph" "ceph-defragosds" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + spec: + template: + metadata: + labels: +{{ tuple $envAll "ceph" "ceph-defragosds" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 12 }} + spec: + serviceAccountName: {{ $serviceAccountName }} + nodeSelector: + {{ .Values.labels.mgr.node_selector_key }}: {{ .Values.labels.mgr.node_selector_value }} + containers: + - name: {{ $serviceAccountName }} +{{ tuple $envAll "ceph_config_helper" | include "helm-toolkit.snippets.image" | indent 12 }} + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: KUBECTL_PARAM + value: {{ tuple $envAll "ceph" "ceph-defragosd" | include "helm-toolkit.snippets.kubernetes_kubectl_params" }} + command: + - /tmp/utils-defragOSDs.sh + - cron + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: pod-etc-ceph + mountPath: /etc/ceph + - name: ceph-client-bin + mountPath: /tmp/utils-defragOSDs.sh + subPath: utils-defragOSDs.sh + readOnly: true + restartPolicy: Never + hostNetwork: true + volumes: + - name: pod-tmp + emptyDir: {} + - name: pod-etc-ceph + emptyDir: {} + - name: ceph-client-bin + configMap: + name: ceph-client-bin + defaultMode: 0555 +{{- end }} diff --git a/ceph-client/templates/deployment-checkdns.yaml b/ceph-client/templates/deployment-checkdns.yaml new file mode 100644 index 0000000000..1adee45229 --- /dev/null +++ b/ceph-client/templates/deployment-checkdns.yaml @@ -0,0 +1,130 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License: is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.deployment_checkdns .Values.deployment.ceph }} +{{- $envAll := . }} + +{{- $serviceAccountName := "ceph-checkdns" }} +{{/* +We will give different name to the RoleBinding resource (see $cephRoleBindingName variable below). +This is neccessary, because the RoleBinding with the default name "ceph-checkdns" exists in the system, +and its reference can not be changed. +*/}} +{{- $cephRoleBindingName := "ceph-checkdns-rolebinding" }} + +{{ tuple $envAll "checkdns" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ printf "%s-%s" $envAll.Release.Name "clusterrole-checkdns" | quote }} +rules: + - apiGroups: + - "" + resources: + - pods + - endpoints + - pods/exec + - namespaces + verbs: + - get + - list + - watch + - create +--- + +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ printf "%s-for-%s" $cephRoleBindingName $envAll.Release.Namespace }} + namespace: {{ $envAll.Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ printf "%s-%s" $envAll.Release.Name "clusterrole-checkdns" | quote }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +--- + +kind: Deployment +apiVersion: apps/v1 +metadata: + name: ceph-checkdns + annotations: + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + labels: +{{ tuple $envAll "ceph" "checkdns" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + selector: + matchLabels: +{{ tuple $envAll "ceph" "checkdns" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} + template: + metadata: + labels: +{{ tuple $envAll "ceph" "checkdns" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} +{{ dict "envAll" $envAll "podName" "ceph-checkdns" "containerNames" (list "ceph-checkdns" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "checkdns" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + affinity: +{{ tuple $envAll "ceph" "checkdns" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} +{{ tuple $envAll "checkdns" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }} + nodeSelector: + {{ .Values.labels.checkdns.node_selector_key }}: {{ .Values.labels.checkdns.node_selector_value }} + initContainers: +{{ tuple $envAll "checkdns" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + hostNetwork: true + dnsPolicy: {{ .Values.pod.dns_policy }} + containers: + - name: ceph-checkdns +{{ tuple $envAll "ceph_config_helper" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.checkdns | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "checkdns" "container" "checkdns" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + env: + - name: CLUSTER + value: "ceph" + - name: K8S_HOST_NETWORK + value: "1" + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: MON_PORT + value: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: MON_PORT_V2 + value: {{ tuple "ceph_mon" "internal" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: KUBECTL_PARAM + value: {{ tuple $envAll "ceph" "checkdns" | include "helm-toolkit.snippets.kubernetes_kubectl_params" }} + command: + - /tmp/_start.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: ceph-client-bin + mountPath: /tmp/_start.sh + subPath: utils-checkDNS_start.sh + readOnly: true + volumes: + - name: pod-tmp + emptyDir: {} + - name: ceph-client-bin + configMap: + name: ceph-client-bin + defaultMode: 0555 +{{- end }} diff --git a/ceph-client/templates/deployment-mds.yaml b/ceph-client/templates/deployment-mds.yaml new file mode 100644 index 0000000000..ba67a8d476 --- /dev/null +++ b/ceph-client/templates/deployment-mds.yaml @@ -0,0 +1,176 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "livenessProbeTemplate" }} +tcpSocket: + port: 6800 +{{- end }} + +{{- define "readinessProbeTemplate" }} +tcpSocket: + port: 6800 +{{- end }} + +{{- if and .Values.manifests.deployment_mds ( and .Values.deployment.ceph .Values.conf.features.mds) }} +{{- $envAll := . }} + +{{- $serviceAccountName := "ceph-mds" }} +{{ tuple $envAll "mds" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: ceph-mds + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "ceph" "mds" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + replicas: {{ .Values.pod.replicas.mds }} + selector: + matchLabels: +{{ tuple $envAll "ceph" "mds" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} +{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }} + template: + metadata: + name: ceph-mds + labels: +{{ tuple $envAll "ceph" "mds" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-etc-client-hash: {{ tuple "configmap-etc-client.yaml" . | include "helm-toolkit.utils.hash" }} +{{ dict "envAll" $envAll "podName" "ceph-mds" "containerNames" (list "ceph-mds" "ceph-init-dirs") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "mds" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + affinity: +{{ tuple $envAll "ceph" "mds" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} +{{ tuple $envAll "mds" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }} + nodeSelector: + {{ .Values.labels.mds.node_selector_key }}: {{ .Values.labels.mds.node_selector_value }} + initContainers: +{{ tuple $envAll "mds" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + - name: ceph-init-dirs +{{ tuple $envAll "ceph_mds" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ dict "envAll" $envAll "application" "mds" "container" "init_dirs" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/init-dirs.sh + env: + - name: CLUSTER + value: "ceph" + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: pod-run + mountPath: /run + - name: pod-etc-ceph + mountPath: /etc/ceph + - name: ceph-client-bin + mountPath: /tmp/init-dirs.sh + subPath: init-dirs.sh + readOnly: true + - name: pod-var-lib-ceph + mountPath: /var/lib/ceph + readOnly: false + - name: pod-var-lib-ceph-crash + mountPath: /var/lib/ceph/crash + readOnly: false + containers: + - name: ceph-mds +{{ tuple $envAll "ceph_mds" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.mds | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "mds" "container" "mds" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/mds-start.sh + env: + - name: CLUSTER + value: "ceph" + - name: CEPHFS_CREATE + value: "1" + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: MON_PORT + value: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: MON_PORT_V2 + value: {{ tuple "ceph_mon" "internal" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + ports: + - containerPort: 6800 +{{ dict "envAll" . "component" "ceph" "container" "ceph-mds" "type" "liveness" "probeTemplate" (include "livenessProbeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | trim | indent 10 }} +{{ dict "envAll" . "component" "ceph" "container" "ceph-mds" "type" "readiness" "probeTemplate" (include "readinessProbeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | trim | indent 10 }} + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: pod-run + mountPath: /run + - name: pod-etc-ceph + mountPath: /etc/ceph + - name: ceph-client-bin + mountPath: /tmp/mds-start.sh + subPath: mds-start.sh + readOnly: true + - name: ceph-client-bin + mountPath: /tmp/utils-checkDNS.sh + subPath: utils-checkDNS.sh + readOnly: true + - name: ceph-client-etc + mountPath: /etc/ceph/ceph.conf.template + subPath: ceph.conf + readOnly: true + - name: ceph-client-admin-keyring + mountPath: /etc/ceph/ceph.client.admin.keyring + subPath: ceph.client.admin.keyring + readOnly: true + - name: ceph-bootstrap-mds-keyring + mountPath: /var/lib/ceph/bootstrap-mds/ceph.keyring + subPath: ceph.keyring + readOnly: false + - name: pod-var-lib-ceph + mountPath: /var/lib/ceph + readOnly: false + - name: pod-var-lib-ceph-crash + mountPath: /var/lib/ceph/crash + readOnly: false + volumes: + - name: pod-tmp + emptyDir: {} + - name: pod-run + emptyDir: + medium: "Memory" + - name: pod-etc-ceph + emptyDir: {} + - name: ceph-client-etc + configMap: + name: ceph-client-etc + defaultMode: 0444 + - name: ceph-client-bin + configMap: + name: ceph-client-bin + defaultMode: 0555 + - name: pod-var-lib-ceph + emptyDir: {} + - name: pod-var-lib-ceph-crash + hostPath: + path: /var/lib/openstack-helm/ceph/crash + type: DirectoryOrCreate + - name: ceph-client-admin-keyring + secret: + secretName: {{ .Values.secrets.keyrings.admin }} + - name: ceph-bootstrap-mds-keyring + secret: + secretName: {{ .Values.secrets.keyrings.mds }} +{{- end }} diff --git a/ceph-client/templates/job-bootstrap.yaml b/ceph-client/templates/job-bootstrap.yaml new file mode 100644 index 0000000000..760ac202f9 --- /dev/null +++ b/ceph-client/templates/job-bootstrap.yaml @@ -0,0 +1,83 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_bootstrap .Values.bootstrap.enabled }} +{{- $envAll := . }} + +{{- $serviceAccountName := "ceph-client-bootstrap" }} +{{ tuple $envAll "bootstrap" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: ceph-client-bootstrap + labels: +{{ tuple $envAll "ceph" "bootstrap" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +spec: + template: + metadata: + labels: +{{ tuple $envAll "ceph" "bootstrap" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} +{{ dict "envAll" $envAll "podName" "ceph-client-bootstrap" "containerNames" (list "ceph-client-bootstrap" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "bootstrap" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} + initContainers: +{{ tuple $envAll "bootstrap" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: ceph-client-bootstrap +{{ tuple $envAll "ceph_bootstrap" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.bootstrap | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "bootstrap" "container" "bootstrap" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/bootstrap.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: ceph-client-bin + mountPath: /tmp/bootstrap.sh + subPath: bootstrap.sh + readOnly: true + - name: ceph-client-etc + mountPath: /etc/ceph/ceph.conf + subPath: ceph.conf + readOnly: true + - name: ceph-client-admin-keyring + mountPath: /etc/ceph/ceph.client.admin.keyring + subPath: ceph.client.admin.keyring + readOnly: true + volumes: + - name: pod-tmp + emptyDir: {} + - name: pod-etc-ceph + emptyDir: {} + - name: ceph-client-bin + configMap: + name: ceph-client-bin + defaultMode: 0555 + - name: ceph-client-etc + configMap: + name: ceph-client-etc + defaultMode: 0444 + - name: ceph-client-admin-keyring + secret: + secretName: {{ .Values.secrets.keyrings.admin }} +{{- end }} diff --git a/ceph-client/templates/job-image-repo-sync.yaml b/ceph-client/templates/job-image-repo-sync.yaml new file mode 100644 index 0000000000..2ffa822b9a --- /dev/null +++ b/ceph-client/templates/job-image-repo-sync.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "ceph-client" -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/ceph-client/templates/job-rbd-pool.yaml b/ceph-client/templates/job-rbd-pool.yaml new file mode 100644 index 0000000000..21d919e8d2 --- /dev/null +++ b/ceph-client/templates/job-rbd-pool.yaml @@ -0,0 +1,117 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_rbd_pool .Values.deployment.ceph }} +{{- $envAll := . }} + +{{- $serviceAccountName := "ceph-rbd-pool" }} +{{ tuple $envAll "rbd_pool" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: ceph-rbd-pool + labels: +{{ tuple $envAll "ceph" "rbd-pool" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +spec: + template: + metadata: + name: ceph-rbd-pool + labels: +{{ tuple $envAll "ceph" "rbd-pool" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ dict "envAll" $envAll "podName" "ceph-rbd-pool" "containerNames" (list "ceph-rbd-pool" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "rbd_pool" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: {{ $envAll.Values.jobs.rbd_pool.restartPolicy | quote }} + affinity: +{{ tuple $envAll "ceph" "rbd-pool" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} + nodeSelector: + {{ $envAll.Values.labels.job.node_selector_key }}: {{ $envAll.Values.labels.job.node_selector_value }} + initContainers: +{{ tuple $envAll "rbd_pool" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: ceph-rbd-pool +{{ tuple $envAll "ceph_rbd_pool" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.rbd_pool | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "rbd_pool" "container" "rbd_pool" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + env: + - name: CLUSTER + value: "ceph" + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: ENABLE_AUTOSCALER + value: {{ .Values.conf.features.pg_autoscaler | quote }} + - name: CLUSTER_SET_FLAGS + value: {{ .Values.conf.features.cluster_flags.set | quote }} + - name: CLUSTER_UNSET_FLAGS + value: {{ .Values.conf.features.cluster_flags.unset | quote }} + command: + - /tmp/pool-init.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: ceph-client-bin + mountPath: /tmp/pool-init.sh + subPath: pool-init.sh + readOnly: true + - name: ceph-client-bin + mountPath: /tmp/pool-calc.py + subPath: pool-calc.py + readOnly: true + - name: pod-etc-ceph + mountPath: /etc/ceph + readOnly: false + - name: ceph-client-etc + mountPath: /etc/ceph/ceph.conf.template + subPath: ceph.conf + readOnly: true + - name: ceph-client-admin-keyring + mountPath: /etc/ceph/ceph.client.admin.keyring + subPath: ceph.client.admin.keyring + readOnly: true + - name: pod-var-lib-ceph + mountPath: /var/lib/ceph + readOnly: false + - name: pod-run + mountPath: /run + readOnly: false + volumes: + - name: pod-tmp + emptyDir: {} + - name: pod-etc-ceph + emptyDir: {} + - name: ceph-client-etc + configMap: + name: ceph-client-etc + defaultMode: 0444 + - name: ceph-client-bin + configMap: + name: ceph-client-bin + defaultMode: 0555 + - name: pod-var-lib-ceph + emptyDir: {} + - name: pod-run + emptyDir: + medium: "Memory" + - name: ceph-client-admin-keyring + secret: + secretName: {{ .Values.secrets.keyrings.admin }} +{{- end }} diff --git a/ceph-client/templates/pod-helm-tests.yaml b/ceph-client/templates/pod-helm-tests.yaml new file mode 100644 index 0000000000..f9117d8e92 --- /dev/null +++ b/ceph-client/templates/pod-helm-tests.yaml @@ -0,0 +1,92 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.helm_tests }} +{{- $envAll := . }} +{{- $serviceAccountName := printf "%s-%s" $envAll.Release.Name "test" }} +{{ tuple $envAll "tests" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: v1 +kind: Pod +metadata: + name: {{ $serviceAccountName }} + labels: +{{ tuple $envAll "ceph-client" "test" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: + "helm.sh/hook": test-success +{{ dict "envAll" $envAll "podName" "ceph-client-test" "containerNames" (list "init" "ceph-cluster-helm-test") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 4 }} +spec: +{{ dict "envAll" $envAll "application" "test" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 2 }} + restartPolicy: Never + serviceAccountName: {{ $serviceAccountName }} + nodeSelector: + {{ .Values.labels.test.node_selector_key }}: {{ .Values.labels.test.node_selector_value }} + initContainers: +{{ tuple $envAll "tests" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 4 }} + containers: + - name: ceph-cluster-helm-test +{{ tuple $envAll "ceph_config_helper" | include "helm-toolkit.snippets.image" | indent 6 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.tests | include "helm-toolkit.snippets.kubernetes_resources" | indent 6 }} +{{ dict "envAll" $envAll "application" "test" "container" "test" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 6 }} + env: + - name: CLUSTER + value: "ceph" + - name: CEPH_DEPLOYMENT_NAMESPACE + value: {{ .Release.Namespace }} + - name: REQUIRED_PERCENT_OF_OSDS + value: {{ .Values.conf.pool.target.required_percent_of_osds | ceil | quote }} + - name: EXPECTED_CRUSHRULE + value: {{ .Values.conf.pool.default.crush_rule | default "replicated_rule" | quote }} + - name: MGR_COUNT + value: {{ .Values.pod.replicas.mgr | default "1" | quote }} + - name: ENABLE_AUTOSCALER + value: {{ .Values.conf.features.pg_autoscaler | quote }} + {{- range $pool := .Values.conf.pool.spec -}} + {{- with $pool }} + - name: {{ .name | upper | replace "." "_" }} + value: {{ .replication | quote }} + {{- end }} + {{- end }} + command: + - /tmp/helm-tests.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: ceph-client-bin + mountPath: /tmp/helm-tests.sh + subPath: helm-tests.sh + readOnly: true + - name: ceph-client-admin-keyring + mountPath: /etc/ceph/ceph.client.admin.keyring + subPath: ceph.client.admin.keyring + readOnly: true + - name: ceph-client-etc + mountPath: /etc/ceph/ceph.conf + subPath: ceph.conf + readOnly: true + volumes: + - name: pod-tmp + emptyDir: {} + - name: ceph-client-bin + configMap: + name: ceph-client-bin + defaultMode: 0555 + - name: ceph-client-admin-keyring + secret: + secretName: {{ .Values.secrets.keyrings.admin }} + - name: ceph-client-etc + configMap: + name: ceph-client-etc + defaultMode: 0444 +{{- end }} diff --git a/ceph-client/templates/secret-registry.yaml b/ceph-client/templates/secret-registry.yaml new file mode 100644 index 0000000000..da979b3223 --- /dev/null +++ b/ceph-client/templates/secret-registry.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_registry .Values.endpoints.oci_image_registry.auth.enabled }} +{{ include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) }} +{{- end }} diff --git a/ceph-client/values.yaml b/ceph-client/values.yaml new file mode 100644 index 0000000000..39ef46f9a1 --- /dev/null +++ b/ceph-client/values.yaml @@ -0,0 +1,603 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for ceph-client. +# This is a YAML-formatted file. +# Declare name/value pairs to be passed into your templates. +# name: value + +--- +deployment: + ceph: true + +release_group: null + +images: + pull_policy: IfNotPresent + tags: + ceph_bootstrap: 'docker.io/openstackhelm/ceph-daemon:ubuntu_jammy_19.2.1-1-20250207' + ceph_config_helper: 'docker.io/openstackhelm/ceph-config-helper:ubuntu_jammy_19.2.1-1-20250207' + ceph_mds: 'docker.io/openstackhelm/ceph-daemon:ubuntu_jammy_19.2.1-1-20250207' + ceph_rbd_pool: 'docker.io/openstackhelm/ceph-config-helper:ubuntu_jammy_19.2.1-1-20250207' + dep_check: 'quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal' + image_repo_sync: 'docker.io/library/docker:17.07.0' + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +labels: + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + test: + node_selector_key: openstack-control-plane + node_selector_value: enabled + mgr: + node_selector_key: ceph-mgr + node_selector_value: enabled + mds: + node_selector_key: ceph-mds + node_selector_value: enabled + checkdns: + node_selector_key: ceph-mon + node_selector_value: enabled + +pod: + security_context: + checkdns: + pod: + runAsUser: 65534 + container: + checkdns: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + mds: + pod: + runAsUser: 65534 + container: + init_dirs: + runAsUser: 0 + readOnlyRootFilesystem: true + mds: + runAsUser: 64045 + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + bootstrap: + pod: + runAsUser: 65534 + container: + bootstrap: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + rbd_pool: + pod: + runAsUser: 65534 + container: + rbd_pool: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + test: + pod: + runAsUser: 65534 + container: + test: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + dns_policy: "ClusterFirstWithHostNet" + replicas: + mds: 2 + lifecycle: + upgrades: + deployments: + pod_replacement_strategy: RollingUpdate + revision_history: 3 + rolling_update: + max_surge: 25% + max_unavailable: 25% + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + weight: + default: 10 + resources: + enabled: false + mds: + requests: + memory: "10Mi" + cpu: "250m" + limits: + memory: "50Mi" + cpu: "500m" + checkdns: + requests: + memory: "5Mi" + cpu: "250m" + limits: + memory: "50Mi" + cpu: "500m" + jobs: + bootstrap: + limits: + memory: "1024Mi" + cpu: "2000m" + requests: + memory: "128Mi" + cpu: "500m" + image_repo_sync: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + rbd_pool: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + tests: + requests: + memory: "10Mi" + cpu: "250m" + limits: + memory: "50Mi" + cpu: "500m" + tolerations: + checkdns: + tolerations: + - effect: NoExecute + key: node.kubernetes.io/not-ready + operator: Exists + tolerationSeconds: 60 + - effect: NoExecute + key: node.kubernetes.io/unreachable + operator: Exists + tolerationSeconds: 60 + mds: + tolerations: + - effect: NoExecute + key: node.kubernetes.io/not-ready + operator: Exists + tolerationSeconds: 60 + - effect: NoExecute + key: node.kubernetes.io/unreachable + operator: Exists + tolerationSeconds: 60 + probes: + ceph: + ceph-mds: + readiness: + enabled: true + params: + timeoutSeconds: 5 + liveness: + enabled: true + params: + initialDelaySeconds: 60 + timeoutSeconds: 5 + +secrets: + keyrings: + mon: ceph-mon-keyring + mds: ceph-bootstrap-mds-keyring + osd: ceph-bootstrap-osd-keyring + rgw: ceph-bootstrap-rgw-keyring + mgr: ceph-bootstrap-mgr-keyring + admin: ceph-client-admin-keyring + oci_image_registry: + ceph-client: ceph-client-oci-image-registry + +network: + public: 192.168.0.0/16 + cluster: 192.168.0.0/16 + +jobs: + ceph_defragosds: + # Execute the 1st of each month + cron: "0 0 1 * *" + history: + # Number of successful job to keep + successJob: 1 + # Number of failed job to keep + failJob: 1 + concurrency: + # Skip new job if previous job still active + execPolicy: Forbid + startingDeadlineSecs: 60 + pool_checkPGs: + # Execute every 15 minutes + cron: "*/15 * * * *" + history: + # Number of successful job to keep + successJob: 1 + # Number of failed job to keep + failJob: 1 + concurrency: + # Skip new job if previous job still active + execPolicy: Forbid + startingDeadlineSecs: 60 + rbd_pool: + restartPolicy: OnFailure + +conf: + features: + mds: true + pg_autoscaler: true + cluster_flags: + # List of flags to set or unset separated by spaces + set: "" + unset: "" + cluster_commands: + # Add additional commands to run against the Ceph cluster here + # NOTE: Beginning with Pacific, mon_allow_pool_size_one must be + # configured here to allow gate scripts to use 1x replication. + # Adding it to /etc/ceph/ceph.conf doesn't seem to be effective. + - config set global mon_allow_pool_size_one true + - osd require-osd-release squid + - status + pool: + # NOTE(portdirect): this drives a simple approximation of + # https://ceph.com/pgcalc/, the `target.osd` key should be set to match the + # expected number of osds in a cluster, and the `target.pg_per_osd` should be + # set to match the desired number of placement groups on each OSD. + crush: + # NOTE(portdirect): to use RBD devices with Ubuntu 16.04's 4.4.x series + # kernel this should be set to `hammer` + tunables: null + target: + # NOTE(portdirect): arbitrarily we set the default number of expected OSD's to 5 + # to match the number of nodes in the OSH gate. + osd: 5 + # This the number of OSDs expected in the final state. This is to allow the above + # target to be smaller initially in the event of a partial deployment. This way + # helm tests can still pass at deployment time and pool quotas can be set based on + # the expected final state (actual target quota = final_osd / osd * quota). + final_osd: 5 + # This is just for helm tests to proceed the deployment if we have mentioned % of + # osds are up and running. + required_percent_of_osds: 75 + pg_per_osd: 100 + # NOTE(bw6938): When pools are created with the autoscaler enabled, a pg_num_min + # value specifies the minimum value of pg_num that the autoscaler will target. + # That default was recently changed from 8 to 32 which severely limits the number + # of pools in a small cluster per https://github.com/rook/rook/issues/5091. This change + # overrides the default pg_num_min value of 32 with a value of 8, matching the default + # pg_num value of 8. + pg_num_min: 8 + protected: true + # NOTE(st053q): target quota should be set to the overall cluster full percentage + # to be tolerated as a quota (percent full to allow in order to tolerate some + # level of failure) + # Set target quota to "0" (must be quoted) to remove quotas for all pools + quota: 100 + default: + # NOTE(supamatt): Accepted values are taken from `crush_rules` list. + crush_rule: replicated_rule + crush_rules: + # NOTE(supamatt): Device classes must remain undefined if all OSDs are the + # same device type of backing disks (ie, all HDD or all SDD). + - name: same_host + crush_rule: create-simple + failure_domain: osd + device_class: + - name: replicated_rule + crush_rule: create-simple + failure_domain: host + device_class: + - name: rack_replicated_rule + crush_rule: create-simple + failure_domain: rack + device_class: + # - name: replicated_rule-ssd + # crush_rule: create-replicated + # failure_domain: host + # device_class: sdd + # - name: replicated_rule-hdd + # crush_rule: create-replicated + # failure_domain: host + # device_class: hdd + # - name: rack_replicated_rule-ssd + # crush_rule: create-replicated + # failure_domain: rack + # device_class: ssd + # - name: rack_replicated_rule-hdd + # crush_rule: create-replicated + # failure_domain: rack + # device_class: hdd + # - name: row_replicated_rule + # crush_rule: create-simple + # failure_domain: row + # device_class: + + # NOTE(portdirect): this section describes the pools that will be managed by + # the ceph pool management job, as it tunes the pgs and crush rule, based on + # the above. + spec: + # Health metrics pool + - name: .mgr + application: mgr_devicehealth + replication: 1 + percent_total_data: 5 + # RBD pool + - name: rbd + # An optional "rename" value may be used to change the name of an existing pool. + # If the pool doesn't exist, it will be created and renamed. If the pool exists with + # the original name, it will be renamed. If the pool exists and has already been + # renamed, the name will not be changed. If two pools exist with the two names, the + # pool matching the renamed value will be configured and the other left alone. + # rename: rbd-new + # Optional "delete" and "delete_all_pool_data" values may be used to delete an + # existing pool. Both must exist and must be set to true in order to delete a pool. + # NOTE: Deleting a pool deletes all of its data and is unrecoverable. This is why + # both values are required in order to delete a pool. Neither value does + # anything by itself. + # delete: false + # delete_all_pool_data: false + application: rbd + replication: 3 + percent_total_data: 40 + # Example of 100 GiB pool_quota for rbd pool (no pool quota if absent) + # May be specified in TiB, TB, GiB, GB, MiB, MB, KiB, KB, or bytes + # NOTE: This should always be a string value to avoid Helm issues with large integers + # pool_quota: "100GiB" + # Example of an overridden pg_num_min value for a single pool + # pg_num_min: 32 + # NOTE(supamatt): By default the crush rules used to create each pool will be + # taken from the pool default `crush_rule` unless a pool specific `crush_rule` + # is specified. The rule MUST exist for it to be defined here. + # crush_rule: replicated_rule + # CephFS pools + - name: cephfs_metadata + application: cephfs + replication: 3 + percent_total_data: 5 + - name: cephfs_data + application: cephfs + replication: 3 + percent_total_data: 10 + # RadosGW pools + - name: .rgw.root + application: rgw + replication: 3 + percent_total_data: 0.1 + - name: default.rgw.control + application: rgw + replication: 3 + percent_total_data: 0.1 + - name: default.rgw.data.root + application: rgw + replication: 3 + percent_total_data: 0.1 + - name: default.rgw.gc + application: rgw + replication: 3 + percent_total_data: 0.1 + - name: default.rgw.log + application: rgw + replication: 3 + percent_total_data: 0.1 + - name: default.rgw.intent-log + application: rgw + replication: 3 + percent_total_data: 0.1 + - name: default.rgw.meta + application: rgw + replication: 3 + percent_total_data: 0.1 + - name: default.rgw.usage + application: rgw + replication: 3 + percent_total_data: 0.1 + - name: default.rgw.users.keys + application: rgw + replication: 3 + percent_total_data: 0.1 + - name: default.rgw.users.email + application: rgw + replication: 3 + percent_total_data: 0.1 + - name: default.rgw.users.swift + application: rgw + replication: 3 + percent_total_data: 0.1 + - name: default.rgw.users.uid + application: rgw + replication: 3 + percent_total_data: 0.1 + - name: default.rgw.buckets.extra + application: rgw + replication: 3 + percent_total_data: 0.1 + - name: default.rgw.buckets.index + application: rgw + replication: 3 + percent_total_data: 3 + - name: default.rgw.buckets.data + application: rgw + replication: 3 + percent_total_data: 29 + + ceph: + global: + # auth + cephx: true + cephx_require_signatures: false + cephx_cluster_require_signatures: true + cephx_service_require_signatures: false + objecter_inflight_op_bytes: "1073741824" + objecter_inflight_ops: 10240 + debug_ms: "0/0" + log_file: /dev/stdout + mon_cluster_log_file: /dev/stdout + osd: + osd_mkfs_type: xfs + osd_mkfs_options_xfs: -f -i size=2048 + osd_max_object_name_len: 256 + ms_bind_port_min: 6800 + ms_bind_port_max: 7100 + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - ceph-client-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + bootstrap: + jobs: null + services: + - endpoint: internal + service: ceph_mon + cephfs_client_key_generator: + jobs: null + mds: + jobs: + - ceph-storage-keys-generator + - ceph-mds-keyring-generator + - ceph-rbd-pool + services: + - endpoint: internal + service: ceph_mon + pool_checkpgs: + jobs: + - ceph-rbd-pool + services: + - endpoint: internal + service: ceph_mgr + checkdns: + services: + - endpoint: internal + service: ceph_mon + namespace_client_key_cleaner: + jobs: null + namespace_client_key_generator: + jobs: null + rbd_pool: + services: + - endpoint: internal + service: ceph_mon + - endpoint: internal + service: ceph_mgr + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + tests: + jobs: + - ceph-rbd-pool + - ceph-mgr-keyring-generator + services: + - endpoint: internal + service: ceph_mon + - endpoint: internal + service: ceph_mgr + +bootstrap: + enabled: false + script: | + ceph -s + function ensure_pool () { + ceph osd pool stats $1 || ceph osd pool create $1 $2 + if [[ $(ceph mon versions | awk '/version/{print $3}' | cut -d. -f1) -ge 12 ]]; then + ceph osd pool application enable $1 $3 + fi + } + #ensure_pool volumes 8 cinder + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + ceph-client: + username: ceph-client + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + ceph_mon: + namespace: null + hosts: + default: ceph-mon + discovery: ceph-mon-discovery + host_fqdn_override: + default: null + port: + mon: + default: 6789 + mon_msgr2: + default: 3300 + ceph_mgr: + namespace: null + hosts: + default: ceph-mgr + host_fqdn_override: + default: null + port: + mgr: + default: 7000 + metrics: + default: 9283 + scheme: + default: http + ceph_object_store: + endpoint_namespaces: + - openstack + - ceph + # hosts: + # default: ceph-rgw + # host_fqdn_override: + # default: null + +manifests: + configmap_bin: true + configmap_test_bin: true + configmap_etc: true + deployment_mds: true + deployment_checkdns: true + job_bootstrap: false + job_cephfs_client_key: true + job_image_repo_sync: true + job_rbd_pool: true + helm_tests: true + cronjob_checkPGs: true + cronjob_defragosds: true + secret_registry: true +... diff --git a/ceph-mon/Chart.yaml b/ceph-mon/Chart.yaml new file mode 100644 index 0000000000..038d835da6 --- /dev/null +++ b/ceph-mon/Chart.yaml @@ -0,0 +1,24 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: v1.0.0 +description: OpenStack-Helm Ceph Mon +name: ceph-mon +version: 2024.2.0 +home: https://github.com/ceph/ceph +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit + version: ">= 0.1.0" +... diff --git a/ceph-mon/templates/bin/_bootstrap.sh.tpl b/ceph-mon/templates/bin/_bootstrap.sh.tpl new file mode 100644 index 0000000000..6452d0a073 --- /dev/null +++ b/ceph-mon/templates/bin/_bootstrap.sh.tpl @@ -0,0 +1,18 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +{{ .Values.bootstrap.script | default "echo 'Not Enabled'" }} diff --git a/ceph-mon/templates/bin/_init-dirs.sh.tpl b/ceph-mon/templates/bin/_init-dirs.sh.tpl new file mode 100644 index 0000000000..482a307cc7 --- /dev/null +++ b/ceph-mon/templates/bin/_init-dirs.sh.tpl @@ -0,0 +1,44 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +export LC_ALL=C +: "${HOSTNAME:=$(uname -n)}" +: "${MGR_NAME:=${HOSTNAME}}" +: "${MDS_NAME:=mds-${HOSTNAME}}" +: "${MDS_BOOTSTRAP_KEYRING:=/var/lib/ceph/bootstrap-mds/${CLUSTER}.keyring}" +: "${OSD_BOOTSTRAP_KEYRING:=/var/lib/ceph/bootstrap-osd/${CLUSTER}.keyring}" + +for keyring in ${OSD_BOOTSTRAP_KEYRING} ${MDS_BOOTSTRAP_KEYRING} ; do + mkdir -p "$(dirname "$keyring")" +done + +# Let's create the ceph directories +for DIRECTORY in mon osd mds radosgw tmp mgr crash; do + mkdir -p "/var/lib/ceph/${DIRECTORY}" +done + +# Create socket directory +mkdir -p /run/ceph + +# Create the MDS directory +mkdir -p "/var/lib/ceph/mds/${CLUSTER}-${MDS_NAME}" + +# Create the MGR directory +mkdir -p "/var/lib/ceph/mgr/${CLUSTER}-${MGR_NAME}" + +# Adjust the owner of all those directories +chown -R ceph. /run/ceph/ /var/lib/ceph/* diff --git a/ceph-mon/templates/bin/_post-apply.sh.tpl b/ceph-mon/templates/bin/_post-apply.sh.tpl new file mode 100644 index 0000000000..6659c6f6fe --- /dev/null +++ b/ceph-mon/templates/bin/_post-apply.sh.tpl @@ -0,0 +1,132 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +export LC_ALL=C + +: "${ADMIN_KEYRING:=/etc/ceph/${CLUSTER}.client.admin.keyring}" + +if [[ ! -f /etc/ceph/${CLUSTER}.conf ]]; then + echo "ERROR- /etc/ceph/${CLUSTER}.conf must exist; get it from your existing mon" + exit 1 +fi + +if [[ ! -f ${ADMIN_KEYRING} ]]; then + echo "ERROR- ${ADMIN_KEYRING} must exist; get it from your existing mon" + exit 1 +fi + +ceph --cluster ${CLUSTER} -s +function wait_for_pods() { + timeout=${2:-1800} + end=$(date -ud "${timeout} seconds" +%s) + # Selecting containers with "ceph-mon" name and + # counting them based on "ready" field. + count_pods=".items | map(.status.containerStatuses | .[] | \ + select(.name==\"ceph-mon\")) | \ + group_by(.ready) | map({(.[0].ready | tostring): length}) | .[]" + min_mons="add | if .true >= (.false + .true) \ + then \"pass\" else \"fail\" end" + while true; do + # Leave while loop if all mons are ready. + state=$(kubectl get pods --namespace="${1}" -l component=mon -o json | jq "${count_pods}") + mon_state=$(jq -s "${min_mons}" <<< "${state}") + if [[ "${mon_state}" == \"pass\" ]]; then + break + fi + sleep 5 + + if [ $(date -u +%s) -gt $end ] ; then + echo -e "Containers failed to start after $timeout seconds\n" + kubectl get pods --namespace "${1}" -o wide -l component=mon + exit 1 + fi + done +} + +function check_ds() { + for ds in `kubectl get ds --namespace=$CEPH_NAMESPACE -l component=mon --no-headers=true|awk '{print $1}'` + do + ds_query=`kubectl get ds -n $CEPH_NAMESPACE $ds -o json|jq -r .status` + if echo $ds_query |grep -i "numberAvailable" ;then + currentNumberScheduled=`echo $ds_query|jq -r .currentNumberScheduled` + desiredNumberScheduled=`echo $ds_query|jq -r .desiredNumberScheduled` + numberAvailable=`echo $ds_query|jq -r .numberAvailable` + numberReady=`echo $ds_query|jq -r .numberReady` + updatedNumberScheduled=`echo $ds_query|jq -r .updatedNumberScheduled` + ds_check=`echo "$currentNumberScheduled $desiredNumberScheduled $numberAvailable $numberReady $updatedNumberScheduled"| \ + tr ' ' '\n'|sort -u|wc -l` + if [ $ds_check != 1 ]; then + echo "Some pods in daemonset $ds are not ready" + exit + else + echo "All pods in deamonset $ds are ready" + fi + else + echo "There are no mons under daemonset $ds" + fi + done +} + +function restart_mons() { + mon_pods=`kubectl get po -n $CEPH_NAMESPACE -l component=mon --no-headers | awk '{print $1}'` + + for pod in ${mon_pods} + do + if [[ -n "$pod" ]]; then + echo "Restarting pod $pod" + kubectl delete pod -n $CEPH_NAMESPACE $pod + fi + echo "Waiting for the pod $pod to restart" + # The pod will not be ready in first 60 seconds. Thus we can reduce + # amount of queries to kubernetes. + sleep 60 + wait_for_pods + ceph -s + done +} + +wait_for_pods $CEPH_NAMESPACE + +require_upgrade=0 +max_release=0 + +for ds in `kubectl get ds --namespace=$CEPH_NAMESPACE -l component=mon --no-headers=true|awk '{print $1}'` +do + updatedNumberScheduled=`kubectl get ds -n $CEPH_NAMESPACE $ds -o json|jq -r .status.updatedNumberScheduled` + desiredNumberScheduled=`kubectl get ds -n $CEPH_NAMESPACE $ds -o json|jq -r .status.desiredNumberScheduled` + if [[ $updatedNumberScheduled != $desiredNumberScheduled ]]; then + if kubectl get ds -n $CEPH_NAMESPACE $ds -o json|jq -r .status|grep -i "numberAvailable" ;then + require_upgrade=$((require_upgrade+1)) + _release=`kubectl get ds -n $CEPH_NAMESPACE $ds -o json|jq -r .status.observedGeneration` + max_release=$(( max_release > _release ? max_release : _release )) + fi + fi +done + +echo "Latest revision of the helm chart(s) is : $max_release" + +if [[ "$UNCONDITIONAL_MON_RESTART" == "true" ]] || [[ $max_release -gt 1 ]]; then + if [[ "$UNCONDITIONAL_MON_RESTART" == "true" ]] || [[ $require_upgrade -gt 0 ]]; then + echo "Restart ceph-mon pods one at a time to prevent disruption" + restart_mons + fi + + # Check all the ceph-mon daemonsets + echo "checking DS" + check_ds +else + echo "No revisions found for upgrade" +fi diff --git a/ceph-mon/templates/bin/keys/_bootstrap-keyring-generator.py.tpl b/ceph-mon/templates/bin/keys/_bootstrap-keyring-generator.py.tpl new file mode 100644 index 0000000000..a0a279c7b2 --- /dev/null +++ b/ceph-mon/templates/bin/keys/_bootstrap-keyring-generator.py.tpl @@ -0,0 +1,14 @@ +#!/bin/python +import os +import struct +import time +import base64 +key = os.urandom(16) +header = struct.pack( + ' +create_kube_key $(ceph_gen_key) ${CEPH_KEYRING_NAME} ${CEPH_KEYRING_TEMPLATE} ${KUBE_SECRET_NAME} + +{{ else }} + +echo "Not touching ${KUBE_SECRET_NAME} as this is not the initial deployment" + +{{- end -}} diff --git a/ceph-mon/templates/bin/keys/_storage-keyring-manager.sh.tpl b/ceph-mon/templates/bin/keys/_storage-keyring-manager.sh.tpl new file mode 100644 index 0000000000..431af1ab8a --- /dev/null +++ b/ceph-mon/templates/bin/keys/_storage-keyring-manager.sh.tpl @@ -0,0 +1,100 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +{{ if .Release.IsInstall }} +{{- $envAll := . }} + +function ceph_gen_key () { + python3 ${CEPH_GEN_DIR}/keys-bootstrap-keyring-generator.py +} + +function kube_ceph_keyring_gen () { + CEPH_KEY=$1 + CEPH_KEY_TEMPLATE=$2 + sed "s|{{"{{"}} key {{"}}"}}|${CEPH_KEY}|" ${CEPH_TEMPLATES_DIR}/${CEPH_KEY_TEMPLATE} | base64 -w0 | tr -d '\n' +} + +CEPH_CLIENT_KEY="" +ROOK_CEPH_TOOLS_POD=$(kubectl -n ${DEPLOYMENT_NAMESPACE} get pods --no-headers | awk '/rook-ceph-tools/{print $1}') + +if [[ -n "${ROOK_CEPH_TOOLS_POD}" ]]; then + CEPH_AUTH_KEY_NAME=$(echo "${CEPH_KEYRING_NAME}" | awk -F. '{print $2 "." $3}') + CEPH_CLIENT_KEY=$(kubectl -n ${DEPLOYMENT_NAMESPACE} exec ${ROOK_CEPH_TOOLS_POD} -- ceph auth ls | grep -A1 "${CEPH_AUTH_KEY_NAME}" | awk '/key:/{print $2}') +fi + +if [[ -z "${CEPH_CLIENT_KEY}" ]]; then + CEPH_CLIENT_KEY=$(ceph_gen_key) +fi + +function create_kube_key () { + CEPH_KEYRING=$1 + CEPH_KEYRING_NAME=$2 + CEPH_KEYRING_TEMPLATE=$3 + KUBE_SECRET_NAME=$4 + + if ! kubectl get --namespace ${DEPLOYMENT_NAMESPACE} secrets ${KUBE_SECRET_NAME}; then + { + cat < +create_kube_key ${CEPH_CLIENT_KEY} ${CEPH_KEYRING_NAME} ${CEPH_KEYRING_TEMPLATE} ${CEPH_KEYRING_ADMIN_NAME} + +function create_kube_storage_key () { + CEPH_KEYRING=$1 + KUBE_SECRET_NAME=$2 + + if ! kubectl get --namespace ${DEPLOYMENT_NAMESPACE} secrets ${KUBE_SECRET_NAME}; then + { + cat < +create_kube_storage_key ${CEPH_CLIENT_KEY} ${CEPH_STORAGECLASS_ADMIN_SECRET_NAME} +create_kube_storage_key ${CEPH_CLIENT_KEY} ${CEPH_STORAGECLASS_ADMIN_SECRET_NAME_NODE} + +{{ else }} + +echo "Not touching ${KUBE_SECRET_NAME} as this is not the initial deployment" + +{{ end }} diff --git a/ceph-mon/templates/bin/mgr/_check.sh.tpl b/ceph-mon/templates/bin/mgr/_check.sh.tpl new file mode 100644 index 0000000000..e37f2d084b --- /dev/null +++ b/ceph-mon/templates/bin/mgr/_check.sh.tpl @@ -0,0 +1,42 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +export LC_ALL=C + +COMMAND="${@:-liveness}" + +function heath_check () { + ASOK=$(ls /var/run/ceph/${CLUSTER}-mgr*) + MGR_NAME=$(basename ${ASOK} | sed -e 's/.asok//' | cut -f 1 -d '.' --complement) + MGR_STATE=$(ceph --cluster ${CLUSTER} --connect-timeout 1 daemon mgr.${MGR_NAME} status|grep "osd_epoch") + if [ $? = 0 ]; then + exit 0 + else + echo $MGR_STATE + exit 1 + fi +} + +function liveness () { + heath_check +} + +function readiness () { + heath_check +} + +$COMMAND diff --git a/ceph-mon/templates/bin/mgr/_start.sh.tpl b/ceph-mon/templates/bin/mgr/_start.sh.tpl new file mode 100644 index 0000000000..d05175cd16 --- /dev/null +++ b/ceph-mon/templates/bin/mgr/_start.sh.tpl @@ -0,0 +1,79 @@ +#!/bin/bash +set -ex +: "${CEPH_GET_ADMIN_KEY:=0}" +: "${MGR_NAME:=$(uname -n)}" +: "${MGR_KEYRING:=/var/lib/ceph/mgr/${CLUSTER}-${MGR_NAME}/keyring}" +: "${ADMIN_KEYRING:=/etc/ceph/${CLUSTER}.client.admin.keyring}" +: "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}" + +{{ include "helm-toolkit.snippets.mon_host_from_k8s_ep" . }} + +if [[ ! -e ${CEPH_CONF}.template ]]; then + echo "ERROR- ${CEPH_CONF}.template must exist; get it from your existing mon" + exit 1 +else + ENDPOINT=$(mon_host_from_k8s_ep "${NAMESPACE}" ceph-mon-discovery) + if [[ "${ENDPOINT}" == "" ]]; then + /bin/sh -c -e "cat ${CEPH_CONF}.template | tee ${CEPH_CONF}" || true + else + /bin/sh -c -e "cat ${CEPH_CONF}.template | sed 's#mon_host.*#mon_host = ${ENDPOINT}#g' | tee ${CEPH_CONF}" || true + fi +fi + +if [ ${CEPH_GET_ADMIN_KEY} -eq 1 ]; then + if [[ ! -e ${ADMIN_KEYRING} ]]; then + echo "ERROR- ${ADMIN_KEYRING} must exist; get it from your existing mon" + exit 1 + fi +fi + +# Create a MGR keyring +rm -rf $MGR_KEYRING +if [ ! -e "$MGR_KEYRING" ]; then + # Create ceph-mgr key + timeout 10 ceph --cluster "${CLUSTER}" auth get-or-create mgr."${MGR_NAME}" mon 'allow profile mgr' osd 'allow *' mds 'allow *' -o "$MGR_KEYRING" + chown --verbose ceph. "$MGR_KEYRING" + chmod 600 "$MGR_KEYRING" +fi + +echo "SUCCESS" + +ceph --cluster "${CLUSTER}" -v + +# Env. variables matching the pattern "_" will be +# found and parsed for config-key settings by +# ceph config set mgr mgr// +MODULES_TO_DISABLE=`ceph mgr dump | python3 -c "import json, sys; print(' '.join(json.load(sys.stdin)['modules']))"` + +for module in ${ENABLED_MODULES}; do + # This module may have been enabled in the past + # remove it from the disable list if present + MODULES_TO_DISABLE=${MODULES_TO_DISABLE/$module/} + + options=`env | grep ^${module}_ || true` + for option in ${options}; do + #strip module name + option=${option/${module}_/} + key=`echo $option | cut -d= -f1` + value=`echo $option | cut -d= -f2` + if [[ $(ceph mon versions | awk '/version/{print $3}' | cut -d. -f1) -ge 14 ]]; then + ceph --cluster "${CLUSTER}" config set mgr mgr/$module/$key $value --force + else + ceph --cluster "${CLUSTER}" config set mgr mgr/$module/$key $value + fi + done + ceph --cluster "${CLUSTER}" mgr module enable ${module} --force +done + +for module in $MODULES_TO_DISABLE; do + ceph --cluster "${CLUSTER}" mgr module disable ${module} +done + +echo "SUCCESS" +# start ceph-mgr +exec /usr/bin/ceph-mgr \ + --cluster "${CLUSTER}" \ + --setuser "ceph" \ + --setgroup "ceph" \ + -d \ + -i "${MGR_NAME}" diff --git a/ceph-mon/templates/bin/mon/_check.sh.tpl b/ceph-mon/templates/bin/mon/_check.sh.tpl new file mode 100644 index 0000000000..7a7d1c663d --- /dev/null +++ b/ceph-mon/templates/bin/mon/_check.sh.tpl @@ -0,0 +1,61 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +COMMAND="${@:-liveness}" +: ${K8S_HOST_NETWORK:=0} + +function heath_check () { + SOCKDIR=${CEPH_SOCKET_DIR:-/run/ceph} + SBASE=${CEPH_OSD_SOCKET_BASE:-ceph-mon} + SSUFFIX=${CEPH_SOCKET_SUFFIX:-asok} + + MON_ID=$(ps auwwx | grep ceph-mon | grep -v "$1" | grep -v grep | sed 's/.*-i\ //;s/\ .*//'|awk '{print $1}') + + if [ -z "${MON_ID}" ]; then + if [[ ${K8S_HOST_NETWORK} -eq 0 ]]; then + MON_NAME=${POD_NAME} + else + MON_NAME=${NODE_NAME} + fi + fi + + if [ -S "${SOCKDIR}/${SBASE}.${MON_NAME}.${SSUFFIX}" ]; then + MON_STATE=$(ceph -f json-pretty --connect-timeout 1 --admin-daemon "${SOCKDIR}/${SBASE}.${MON_NAME}.${SSUFFIX}" mon_status|grep state|sed 's/.*://;s/[^a-z]//g') + echo "MON ${MON_ID} ${MON_STATE}"; + # this might be a stricter check than we actually want. what are the + # other values for the "state" field? + for S in ${MON_LIVE_STATE}; do + if [ "x${MON_STATE}x" = "x${S}x" ]; then + exit 0 + fi + done + fi + # if we made it this far, things are not running + exit 1 +} + +function liveness () { + MON_LIVE_STATE="probing electing synchronizing leader peon" + heath_check +} + +function readiness () { + MON_LIVE_STATE="leader peon" + heath_check +} + +$COMMAND diff --git a/ceph-mon/templates/bin/mon/_start.sh.tpl b/ceph-mon/templates/bin/mon/_start.sh.tpl new file mode 100644 index 0000000000..739ac60b30 --- /dev/null +++ b/ceph-mon/templates/bin/mon/_start.sh.tpl @@ -0,0 +1,114 @@ +#!/bin/bash +set -ex +export LC_ALL=C +: "${K8S_HOST_NETWORK:=0}" +: "${MON_KEYRING:=/etc/ceph/${CLUSTER}.mon.keyring}" +: "${ADMIN_KEYRING:=/etc/ceph/${CLUSTER}.client.admin.keyring}" +: "${MDS_BOOTSTRAP_KEYRING:=/var/lib/ceph/bootstrap-mds/${CLUSTER}.keyring}" +: "${OSD_BOOTSTRAP_KEYRING:=/var/lib/ceph/bootstrap-osd/${CLUSTER}.keyring}" +: "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}" + +{{ include "helm-toolkit.snippets.mon_host_from_k8s_ep" . }} + +if [[ ! -e ${CEPH_CONF}.template ]]; then + echo "ERROR- ${CEPH_CONF}.template must exist; get it from your existing mon" + exit 1 +else + + ENDPOINT=$(mon_host_from_k8s_ep "${NAMESPACE}" ceph-mon-discovery) + + if [[ -z "${ENDPOINT}" ]]; then + /bin/sh -c -e "cat ${CEPH_CONF}.template | tee ${CEPH_CONF}" || true + else + /bin/sh -c -e "cat ${CEPH_CONF}.template | sed 's#mon_host.*#mon_host = ${ENDPOINT}#g' | tee ${CEPH_CONF}" || true + fi +fi + +if [[ -z "$CEPH_PUBLIC_NETWORK" ]]; then + echo "ERROR- CEPH_PUBLIC_NETWORK must be defined as the name of the network for the OSDs" + exit 1 +fi + +if [[ -z "$MON_IP" ]]; then + echo "ERROR- MON_IP must be defined as the IP address of the monitor" + exit 1 +fi + +if [[ ${K8S_HOST_NETWORK} -eq 0 ]]; then + MON_NAME=${POD_NAME} +else + MON_NAME=${NODE_NAME} +fi +MON_DATA_DIR="/var/lib/ceph/mon/${CLUSTER}-${MON_NAME}" +MONMAP="/etc/ceph/monmap-${CLUSTER}" + +# Make the monitor directory +/bin/sh -c "mkdir -p \"${MON_DATA_DIR}\"" + +function get_mon_config { + # Get fsid from ceph.conf + local fsid=$(ceph-conf --lookup fsid -c /etc/ceph/${CLUSTER}.conf) + + timeout=10 + MONMAP_ADD="" + + while [[ -z "${MONMAP_ADD// }" && "${timeout}" -gt 0 ]]; do + # Get the ceph mon pods (name and IP) from the Kubernetes API. Formatted as a set of monmap params + if [[ ${K8S_HOST_NETWORK} -eq 0 ]]; then + MONMAP_ADD=$(kubectl get pods --namespace=${NAMESPACE} ${KUBECTL_PARAM} -o template --template="{{`{{range .items}}`}}{{`{{if .status.podIP}}`}}--addv {{`{{.metadata.name}}`}} [v1:{{`{{.status.podIP}}`}}:${MON_PORT},v2:{{`{{.status.podIP}}`}}:${MON_PORT_V2}] {{`{{end}}`}} {{`{{end}}`}}") + else + MONMAP_ADD=$(kubectl get pods --namespace=${NAMESPACE} ${KUBECTL_PARAM} -o template --template="{{`{{range .items}}`}}{{`{{if .status.podIP}}`}}--addv {{`{{.spec.nodeName}}`}} [v1:{{`{{.status.podIP}}`}}:${MON_PORT},v2:{{`{{.status.podIP}}`}}:${MON_PORT_V2}] {{`{{end}}`}} {{`{{end}}`}}") + fi + (( timeout-- )) + sleep 1 + done + + if [[ -z "${MONMAP_ADD// }" ]]; then + exit 1 + fi + + # Create a monmap with the Pod Names and IP + monmaptool --create ${MONMAP_ADD} --fsid ${fsid} ${MONMAP} --clobber +} + +get_mon_config + +# If we don't have a monitor keyring, this is a new monitor +if [ ! -e "${MON_DATA_DIR}/keyring" ]; then + if [ ! -e ${MON_KEYRING}.seed ]; then + echo "ERROR- ${MON_KEYRING}.seed must exist. You can extract it from your current monitor by running 'ceph auth get mon. -o ${MON_KEYRING}' or use a KV Store" + exit 1 + else + cp -vf ${MON_KEYRING}.seed ${MON_KEYRING} + fi + + if [ ! -e ${MONMAP} ]; then + echo "ERROR- ${MONMAP} must exist. You can extract it from your current monitor by running 'ceph mon getmap -o ${MONMAP}' or use a KV Store" + exit 1 + fi + + # Testing if it's not the first monitor, if one key doesn't exist we assume none of them exist + for KEYRING in ${OSD_BOOTSTRAP_KEYRING} ${MDS_BOOTSTRAP_KEYRING} ${ADMIN_KEYRING}; do + ceph-authtool ${MON_KEYRING} --import-keyring ${KEYRING} + done + + # Prepare the monitor daemon's directory with the map and keyring + ceph-mon --setuser ceph --setgroup ceph --cluster "${CLUSTER}" --mkfs -i ${MON_NAME} --inject-monmap ${MONMAP} --keyring ${MON_KEYRING} --mon-data "${MON_DATA_DIR}" +else + echo "Trying to get the most recent monmap..." + # Ignore when we timeout, in most cases that means the cluster has no quorum or + # no mons are up and running yet + timeout 5 ceph --cluster "${CLUSTER}" mon getmap -o ${MONMAP} || true + ceph-mon --setuser ceph --setgroup ceph --cluster "${CLUSTER}" -i ${MON_NAME} --inject-monmap ${MONMAP} --keyring ${MON_KEYRING} --mon-data "${MON_DATA_DIR}" + timeout 7 ceph --cluster "${CLUSTER}" mon add "${MON_NAME}" "${MON_IP}:${MON_PORT_V2}" || true +fi + +# start MON +exec /usr/bin/ceph-mon \ + --cluster "${CLUSTER}" \ + --setuser "ceph" \ + --setgroup "ceph" \ + -d \ + -i ${MON_NAME} \ + --mon-data "${MON_DATA_DIR}" \ + --public-addr "${MON_IP}:${MON_PORT_V2}" diff --git a/ceph-mon/templates/bin/mon/_stop.sh.tpl b/ceph-mon/templates/bin/mon/_stop.sh.tpl new file mode 100644 index 0000000000..3f564e88d5 --- /dev/null +++ b/ceph-mon/templates/bin/mon/_stop.sh.tpl @@ -0,0 +1,14 @@ +#!/bin/bash + +set -ex + +NUMBER_OF_MONS=$(ceph mon stat | awk '$3 == "mons" {print $2}') +if [[ "${NUMBER_OF_MONS}" -gt "3" ]]; then + if [[ ${K8S_HOST_NETWORK} -eq 0 ]]; then + ceph mon remove "${POD_NAME}" + else + ceph mon remove "${NODE_NAME}" + fi +else + echo "doing nothing since we are running less than or equal to 3 mons" +fi diff --git a/ceph-mon/templates/bin/moncheck/_reap-zombies.py.tpl b/ceph-mon/templates/bin/moncheck/_reap-zombies.py.tpl new file mode 100644 index 0000000000..36b00356a7 --- /dev/null +++ b/ceph-mon/templates/bin/moncheck/_reap-zombies.py.tpl @@ -0,0 +1,50 @@ +#!/usr/bin/python +import re +import os +import subprocess # nosec +import json + +MON_REGEX = r"^\d: \[((v\d+:([0-9\.]*):\d+\/\d+,*)+)] mon.([^ ]*)$" +# kubctl_command = 'kubectl get pods --namespace=${NAMESPACE} -l component=mon,application=ceph -o template --template="{ {{"}}"}}range .items{{"}}"}} \\"{{"}}"}}.metadata.name{{"}}"}}\\": \\"{{"}}"}}.status.podIP{{"}}"}}\\" , {{"}}"}}end{{"}}"}} }"' +if int(os.getenv('K8S_HOST_NETWORK', 0)) > 0: + kubectl_command = 'kubectl get pods --namespace=${NAMESPACE} -l component=mon,application=ceph -o template --template="{ {{"{{"}}range \$i, \$v := .items{{"}}"}} {{"{{"}} if \$i{{"}}"}} , {{"{{"}} end {{"}}"}} \\"{{"{{"}}\$v.spec.nodeName{{"}}"}}\\": \\"{{"{{"}}\$v.status.podIP{{"}}"}}\\" {{"{{"}}end{{"}}"}} }"' +else: + kubectl_command = 'kubectl get pods --namespace=${NAMESPACE} -l component=mon,application=ceph -o template --template="{ {{"{{"}}range \$i, \$v := .items{{"}}"}} {{"{{"}} if \$i{{"}}"}} , {{"{{"}} end {{"}}"}} \\"{{"{{"}}\$v.metadata.name{{"}}"}}\\": \\"{{"{{"}}\$v.status.podIP{{"}}"}}\\" {{"{{"}}end{{"}}"}} }"' + +monmap_command = "ceph --cluster=${CLUSTER} mon getmap > /tmp/monmap && monmaptool -f /tmp/monmap --print" + + +def extract_mons_from_monmap(): + monmap = subprocess.check_output(monmap_command, shell=True).decode('utf-8') # nosec + mons = {} + for line in monmap.split("\n"): + m = re.match(MON_REGEX, line) + if m is not None: + mons[m.group(4)] = m.group(3) + return mons + +def extract_mons_from_kubeapi(): + kubemap = subprocess.check_output(kubectl_command, shell=True).decode('utf-8') # nosec + return json.loads(kubemap) + +current_mons = extract_mons_from_monmap() +expected_mons = extract_mons_from_kubeapi() + +print("current mons: %s" % current_mons) +print("expected mons: %s" % expected_mons) + +removed_mon = False +for mon in current_mons: + if not mon in expected_mons: + print("removing zombie mon %s" % mon) + subprocess.call(["ceph", "--cluster", os.environ["NAMESPACE"], "mon", "remove", mon]) # nosec + removed_mon = True + elif current_mons[mon] != expected_mons[mon]: # check if for some reason the ip of the mon changed + print("ip change detected for pod %s" % mon) + subprocess.call(["kubectl", "--namespace", os.environ["NAMESPACE"], "delete", "pod", mon]) # nosec + removed_mon = True + print("deleted mon %s via the kubernetes api" % mon) + + +if not removed_mon: + print("no zombie mons found ...") diff --git a/ceph-mon/templates/bin/moncheck/_start.sh.tpl b/ceph-mon/templates/bin/moncheck/_start.sh.tpl new file mode 100644 index 0000000000..f1f5fcd08f --- /dev/null +++ b/ceph-mon/templates/bin/moncheck/_start.sh.tpl @@ -0,0 +1,74 @@ +#!/bin/bash +set -ex +export LC_ALL=C +: "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}" + +{{ include "helm-toolkit.snippets.mon_host_from_k8s_ep" . }} + +if [[ ! -e ${CEPH_CONF}.template ]]; then + echo "ERROR- ${CEPH_CONF}.template must exist; get it from your existing mon" + exit 1 +else + ENDPOINT=$(mon_host_from_k8s_ep ${NAMESPACE} ceph-mon-discovery) + if [[ "${ENDPOINT}" == "" ]]; then + /bin/sh -c -e "cat ${CEPH_CONF}.template | tee ${CEPH_CONF}" || true + else + /bin/sh -c -e "cat ${CEPH_CONF}.template | sed 's#mon_host.*#mon_host = ${ENDPOINT}#g' | tee ${CEPH_CONF}" || true + fi +fi + +function check_mon_msgr2 { + if [[ $(ceph mon versions | awk '/version/{print $3}' | cut -d. -f1) -ge 14 ]]; then + if ceph health detail|grep -i "MON_MSGR2_NOT_ENABLED"; then + echo "ceph-mon msgr v2 not enabled on all ceph mons so enabling" + ceph mon enable-msgr2 + fi + fi +} + +function get_mon_count { + ceph mon count-metadata hostname | jq '. | length' +} + +function check_mon_addrs { + local mon_dump=$(ceph mon dump) + local mon_hostnames=$(echo "${mon_dump}" | awk '/mon\./{print $3}' | sed 's/mon\.//g') + local mon_endpoints=$(kubectl get endpoints ceph-mon-discovery -n ${NAMESPACE} -o json) + local v1_port=$(jq '.subsets[0].ports[] | select(.name == "mon") | .port' <<< ${mon_endpoints}) + local v2_port=$(jq '.subsets[0].ports[] | select(.name == "mon-msgr2") | .port' <<< ${mon_endpoints}) + + for mon in ${mon_hostnames}; do + local mon_endpoint=$(echo "${mon_dump}" | awk "/${mon}/{print \$2}") + local mon_ip=$(jq -r ".subsets[0].addresses[] | select(.nodeName == \"${mon}\") | .ip" <<< ${mon_endpoints}) + + # Skip this mon if it doesn't appear in the list of kubernetes endpoints + if [[ -n "${mon_ip}" ]]; then + local desired_endpoint=$(printf '[v1:%s:%s/0,v2:%s:%s/0]' ${mon_ip} ${v1_port} ${mon_ip} ${v2_port}) + + if [[ "${mon_endpoint}" != "${desired_endpoint}" ]]; then + echo "endpoint for ${mon} is ${mon_endpoint}, setting it to ${desired_endpoint}" + ceph mon set-addrs ${mon} ${desired_endpoint} + fi + fi + done +} + +function watch_mon_health { + previous_mon_count=$(get_mon_count) + while [ true ]; do + mon_count=$(get_mon_count) + if [[ ${mon_count} -ne ${previous_mon_count} ]]; then + echo "checking for zombie mons" + python3 /tmp/moncheck-reap-zombies.py || true + fi + previous_mon_count=${mon_count} + echo "checking for ceph-mon msgr v2" + check_mon_msgr2 + echo "checking mon endpoints in monmap" + check_mon_addrs + echo "sleep 30 sec" + sleep 30 + done +} + +watch_mon_health diff --git a/ceph-mon/templates/bin/utils/_checkDNS.sh.tpl b/ceph-mon/templates/bin/utils/_checkDNS.sh.tpl new file mode 100644 index 0000000000..b7e360b2fe --- /dev/null +++ b/ceph-mon/templates/bin/utils/_checkDNS.sh.tpl @@ -0,0 +1,38 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +: "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}" +ENDPOINT="{$1}" + +function check_mon_dns () { + GREP_CMD=$(grep -rl 'ceph-mon' ${CEPH_CONF}) + + if [[ "${ENDPOINT}" == "{up}" ]]; then + echo "If DNS is working, we are good here" + elif [[ "${ENDPOINT}" != "" ]]; then + if [[ ${GREP_CMD} != "" ]]; then + # No DNS, write CEPH MONs IPs into ${CEPH_CONF} + sh -c -e "cat ${CEPH_CONF}.template | sed 's/mon_host.*/mon_host = ${ENDPOINT}/g' | tee ${CEPH_CONF}" > /dev/null 2>&1 + else + echo "endpoints are already cached in ${CEPH_CONF}" + exit + fi + fi +} + +check_mon_dns + +exit diff --git a/ceph-mon/templates/bin/utils/_checkObjectReplication.py.tpl b/ceph-mon/templates/bin/utils/_checkObjectReplication.py.tpl new file mode 100755 index 0000000000..af0ae45808 --- /dev/null +++ b/ceph-mon/templates/bin/utils/_checkObjectReplication.py.tpl @@ -0,0 +1,31 @@ +#!/usr/bin/python3 + +import subprocess # nosec +import json +import sys +import collections + +if (int(len(sys.argv)) == 1): + print("Please provide pool name to test , example: checkObjectReplication.py ") + sys.exit(1) +else: + poolName = sys.argv[1] + cmdRep = 'ceph osd map' + ' ' + str(poolName) + ' ' + 'testreplication -f json-pretty' + objectRep = subprocess.check_output(cmdRep, shell=True) # nosec + repOut = json.loads(objectRep) + osdNumbers = repOut['up'] + print("Test object got replicated on these osds: %s" % str(osdNumbers)) + + osdHosts= [] + for osd in osdNumbers: + cmdFind = 'ceph osd find' + ' ' + str(osd) + osdFind = subprocess.check_output(cmdFind , shell=True) # nosec + osdHost = json.loads(osdFind) + osdHostLocation = osdHost['crush_location'] + osdHosts.append(osdHostLocation['host']) + + print("Test object got replicated on these hosts: %s" % str(osdHosts)) + + print("Hosts hosting multiple copies of a placement groups are: %s" % + str([item for item, count in collections.Counter(osdHosts).items() if count > 1])) + sys.exit(0) diff --git a/ceph-mon/templates/bin/utils/_checkPGs.py.tpl b/ceph-mon/templates/bin/utils/_checkPGs.py.tpl new file mode 100755 index 0000000000..9836b7cccf --- /dev/null +++ b/ceph-mon/templates/bin/utils/_checkPGs.py.tpl @@ -0,0 +1,263 @@ +#!/usr/bin/python + +import subprocess # nosec +import json +import sys +from argparse import * + +class cephCRUSH(): + """ + Currently, this script is coded to work with the ceph clusters that have + these type-ids -- osd, host, rack, root. To add other type_ids to the + CRUSH map, this script needs enhancements to include the new type_ids. + + type_id name + ------- ---- + 0 osd + 1 host + 2 chassis + 3 rack + 4 row + 5 pdu + 6 pod + 7 room + 8 datacenter + 9 region + 10 root + + Ceph organizes the CRUSH map in hierarchical topology. At the top, it is + the root. The next levels are racks, hosts, and OSDs, respectively. The + OSDs are at the leaf level. This script looks at OSDs in each placement + group of a ceph pool. For each OSD, starting from the OSD leaf level, this + script traverses up to the root. Along the way, the host and rack are + recorded and then verified to make sure the paths to the root are in + separate failure domains. This script reports the offending PGs to stdout. + """ + + """ + This list stores the ceph crush hierarchy retrieved from the + ceph osd crush tree -f json-pretty + """ + crushHierarchy = [] + + """ + Failure Domains - currently our crush map uses these type IDs - osd, + host, rack, root + If we need to add chassis type (or other types) later on, add the + type to the if statement in the crushFD construction section. + + crushFD[0] = {'id': -2, 'name': 'host1', 'type': 'host'} + crushFD[23] = {'id': -5, 'name': 'host2', 'type': 'host'} + crushFD[68] = {'id': -7, 'name': 'host3', 'type': 'host'} + rack_FD[-2] = {'id': -9, 'name': 'rack1', 'type': 'rack' } + rack_FD[-15] = {'id': -17, 'name': 'rack2', 'type': 'rack' } + root_FD[-17] = {'id': -1, 'name': 'default', 'type': 'root' }} + root_FD[-9] = {'id': -1, 'name': 'default', 'type': 'root' }} + """ + crushFD = {} + + def __init__(self, poolName): + if 'all' in poolName or 'All' in poolName: + try: + poolLs = 'ceph osd pool ls -f json-pretty' + poolstr = subprocess.check_output(poolLs, shell=True) # nosec + self.listPoolName = json.loads(poolstr) + except subprocess.CalledProcessError as e: + print('{}'.format(e)) + """Unable to get all pools - cannot proceed""" + sys.exit(2) + else: + self.listPoolName = poolName + + try: + """Retrieve the crush hierarchies""" + crushTree = "ceph osd crush tree -f json-pretty | jq .nodes" + chstr = subprocess.check_output(crushTree, shell=True) # nosec + self.crushHierarchy = json.loads(chstr) + except subprocess.CalledProcessError as e: + print('{}'.format(e)) + """Unable to get crush hierarchy - cannot proceed""" + sys.exit(2) + + """ + Number of racks configured in the ceph cluster. The racks that are + present in the crush hierarchy may not be used. The un-used rack + would not show up in the crushFD. + """ + self.count_racks = 0 + + """depth level - 3 is OSD, 2 is host, 1 is rack, 0 is root""" + self.osd_depth = 0 + """Construct the Failure Domains - OSD -> Host -> Rack -> Root""" + for chitem in self.crushHierarchy: + if chitem['type'] == 'host' or \ + chitem['type'] == 'rack' or \ + chitem['type'] == 'root': + for child in chitem['children']: + self.crushFD[child] = {'id': chitem['id'], 'name': chitem['name'], 'type': chitem['type']} + if chitem['type'] == 'rack' and len(chitem['children']) > 0: + self.count_racks += 1 + elif chitem['type'] == 'osd': + if self.osd_depth == 0: + self.osd_depth = chitem['depth'] + + """[ { 'pg-name' : [osd.1, osd.2, osd.3] } ... ]""" + self.poolPGs = [] + """Replica of the pool. Initialize to 0.""" + self.poolSize = 0 + + def isSupportedRelease(self): + cephMajorVer = int(subprocess.check_output("ceph mon versions | awk '/version/{print $3}' | cut -d. -f1", shell=True)) # nosec + return cephMajorVer >= 14 + + def getPoolSize(self, poolName): + """ + size (number of replica) is an attribute of a pool + { "pool": "rbd", "pool_id": 1, "size": 3 } + """ + pSize = {} + """Get the size attribute of the poolName""" + try: + poolGet = 'ceph osd pool get ' + poolName + ' size -f json-pretty' + szstr = subprocess.check_output(poolGet, shell=True) # nosec + pSize = json.loads(szstr) + self.poolSize = pSize['size'] + except subprocess.CalledProcessError as e: + print('{}'.format(e)) + self.poolSize = 0 + """Continue on""" + return + + def checkPGs(self, poolName): + poolPGs = self.poolPGs['pg_stats'] if self.isSupportedRelease() else self.poolPGs + if not poolPGs: + return + print('Checking PGs in pool {} ...'.format(poolName)), + badPGs = False + for pg in poolPGs: + osdUp = pg['up'] + """ + Construct the OSD path from the leaf to the root. If the + replica is set to 3 and there are 3 racks. Each OSD has its + own rack (failure domain). If more than one OSD has the + same rack, this is a violation. If the number of rack is + one, then we need to make sure the hosts for the three OSDs + are different. + """ + check_FD = {} + checkFailed = False + for osd in osdUp: + traverseID = osd + """Start the level with 1 to include the OSD leaf""" + traverseLevel = 1 + while (self.crushFD[traverseID]['type'] != 'root'): + crushType = self.crushFD[traverseID]['type'] + crushName = self.crushFD[traverseID]['name'] + if crushType in check_FD: + check_FD[crushType].append(crushName) + else: + check_FD[crushType] = [crushName] + """traverse up (to the root) one level""" + traverseID = self.crushFD[traverseID]['id'] + traverseLevel += 1 + if not (traverseLevel == self.osd_depth): + raise Exception("OSD depth mismatch") + """ + check_FD should have + { + 'host': ['host1', 'host2', 'host3', 'host4'], + 'rack': ['rack1', 'rack2', 'rack3'] + } + Not checking for the 'root' as there is only one root. + """ + for ktype in check_FD: + kvalue = check_FD[ktype] + if ktype == 'host': + """ + At the host level, every OSD should come from different + host. It is a violation if duplicate hosts are found. + """ + if len(kvalue) != len(set(kvalue)): + if not badPGs: + print('Failed') + badPGs = True + print('OSDs {} in PG {} failed check in host {}'.format(pg['up'], pg['pgid'], kvalue)) + elif ktype == 'rack': + if len(kvalue) == len(set(kvalue)): + continue + else: + """ + There are duplicate racks. This could be due to + situation like pool's size is 3 and there are only + two racks (or one rack). OSDs should come from + different hosts as verified in the 'host' section. + """ + if self.count_racks == len(set(kvalue)): + continue + elif self.count_racks > len(set(kvalue)): + """Not all the racks were used to allocate OSDs""" + if not badPGs: + print('Failed') + badPGs = True + print('OSDs {} in PG {} failed check in rack {}'.format(pg['up'], pg['pgid'], kvalue)) + check_FD.clear() + if not badPGs: + print('Passed') + return + + def checkPoolPGs(self): + for pool in self.listPoolName: + self.getPoolSize(pool) + if self.poolSize == 1: + """No need to check pool with the size set to 1 copy""" + print('Checking PGs in pool {} ... {}'.format(pool, 'Skipped')) + continue + elif self.poolSize == 0: + print('Pool {} was not found.'.format(pool)) + continue + if not self.poolSize > 1: + raise Exception("Pool size was incorrectly set") + + try: + """Get the list of PGs in the pool""" + lsByPool = 'ceph pg ls-by-pool ' + pool + ' -f json-pretty' + pgstr = subprocess.check_output(lsByPool, shell=True) # nosec + self.poolPGs = json.loads(pgstr) + """Check that OSDs in the PG are in separate failure domains""" + self.checkPGs(pool) + except subprocess.CalledProcessError as e: + print('{}'.format(e)) + """Continue to the next pool (if any)""" + return + +def Main(): + parser = ArgumentParser(description=''' +Cross-check the OSDs assigned to the Placement Groups (PGs) of a ceph pool +with the CRUSH topology. The cross-check compares the OSDs in a PG and +verifies the OSDs reside in separate failure domains. PGs with OSDs in +the same failure domain are flagged as violation. The offending PGs are +printed to stdout. + +This CLI is executed on-demand on a ceph-mon pod. To invoke the CLI, you +can specify one pool or list of pools to check. The special pool name +All (or all) checks all the pools in the ceph cluster. +''', + formatter_class=RawTextHelpFormatter) + parser.add_argument('PoolName', type=str, nargs='+', + help='List of pools (or All) to validate the PGs and OSDs mapping') + args = parser.parse_args() + + if ('all' in args.PoolName or + 'All' in args.PoolName) and len(args.PoolName) > 1: + print('You only need to give one pool with special pool All') + sys.exit(1) + + """ + Retrieve the crush hierarchies and store it. Cross-check the OSDs + in each PG searching for failure domain violation. + """ + ccm = cephCRUSH(args.PoolName) + ccm.checkPoolPGs() + +if __name__ == '__main__': + Main() diff --git a/ceph-mon/templates/configmap-bin.yaml b/ceph-mon/templates/configmap-bin.yaml new file mode 100644 index 0000000000..aede9b6af8 --- /dev/null +++ b/ceph-mon/templates/configmap-bin.yaml @@ -0,0 +1,69 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.configmap_bin .Values.deployment.ceph }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ printf "%s-%s" $envAll.Release.Name "bin" | quote }} +data: +{{- if .Values.images.local_registry.active }} + image-repo-sync.sh: | +{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} +{{- end }} + +{{- if .Values.bootstrap.enabled }} + bootstrap.sh: | +{{ tuple "bin/_bootstrap.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} + post-apply.sh: | +{{ tuple "bin/_post-apply.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + + init-dirs.sh: | +{{ tuple "bin/_init-dirs.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + + keys-bootstrap-keyring-generator.py: | +{{ tuple "bin/keys/_bootstrap-keyring-generator.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + keys-bootstrap-keyring-manager.sh: | +{{ tuple "bin/keys/_bootstrap-keyring-manager.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + keys-storage-keyring-manager.sh: | +{{ tuple "bin/keys/_storage-keyring-manager.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + + mon-start.sh: | +{{ tuple "bin/mon/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + mon-stop.sh: | +{{ tuple "bin/mon/_stop.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + mon-check.sh: | +{{ tuple "bin/mon/_check.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + + mgr-start.sh: | +{{ tuple "bin/mgr/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + mgr-check.sh: | +{{ tuple "bin/mgr/_check.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + + moncheck-start.sh: | +{{ tuple "bin/moncheck/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + moncheck-reap-zombies.py: | +{{ tuple "bin/moncheck/_reap-zombies.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + + utils-checkObjectReplication.py: | +{{ tuple "bin/utils/_checkObjectReplication.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + utils-checkDNS.sh: | +{{ tuple "bin/utils/_checkDNS.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} + + utils-checkPGs.py: | +{{ tuple "bin/utils/_checkPGs.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} diff --git a/ceph-mon/templates/configmap-etc.yaml b/ceph-mon/templates/configmap-etc.yaml new file mode 100644 index 0000000000..b8209d3a88 --- /dev/null +++ b/ceph-mon/templates/configmap-etc.yaml @@ -0,0 +1,51 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "ceph.mon.configmap.etc" }} +{{- $configMapName := index . 0 }} +{{- $envAll := index . 1 }} +{{- with $envAll }} + +{{- if .Values.deployment.ceph }} + +{{- if empty .Values.conf.ceph.global.mon_host -}} +{{- $monHost := tuple "ceph_mon" "discovery" "mon_msgr2" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} +{{- $_ := $monHost | set .Values.conf.ceph.global "mon_host" -}} +{{- end -}} + +{{- if empty .Values.conf.ceph.global.fsid -}} +{{- $_ := uuidv4 | set .Values.conf.ceph.global "fsid" -}} +{{- end -}} + +{{- if empty .Values.conf.ceph.osd.cluster_network -}} +{{- $_ := .Values.network.cluster | set .Values.conf.ceph.osd "cluster_network" -}} +{{- end -}} + +{{- if empty .Values.conf.ceph.osd.public_network -}} +{{- $_ := .Values.network.public | set .Values.conf.ceph.osd "public_network" -}} +{{- end -}} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $configMapName }} +data: + ceph.conf: | +{{ include "helm-toolkit.utils.to_ini" .Values.conf.ceph | indent 4 }} +{{- end }} +{{- end }} +{{- end }} +{{- if .Values.manifests.configmap_etc }} +{{- list (printf "%s-%s" .Release.Name "etc") . | include "ceph.mon.configmap.etc" }} +{{- end }} diff --git a/ceph-mon/templates/configmap-templates.yaml b/ceph-mon/templates/configmap-templates.yaml new file mode 100644 index 0000000000..42852ef24f --- /dev/null +++ b/ceph-mon/templates/configmap-templates.yaml @@ -0,0 +1,33 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.configmap_templates .Values.deployment.storage_secrets }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ printf "%s-%s" $envAll.Release.Name "templates" | quote }} +data: + admin.keyring: | +{{ .Values.conf.templates.keyring.admin | indent 4 }} + mon.keyring: | +{{ .Values.conf.templates.keyring.mon | indent 4 }} + bootstrap.keyring.mds: | +{{ .Values.conf.templates.keyring.bootstrap.mds | indent 4 }} + bootstrap.keyring.mgr: | +{{ .Values.conf.templates.keyring.bootstrap.mgr | indent 4 }} + bootstrap.keyring.osd: | +{{ .Values.conf.templates.keyring.bootstrap.osd | indent 4 }} +{{- end }} diff --git a/ceph-mon/templates/daemonset-mon.yaml b/ceph-mon/templates/daemonset-mon.yaml new file mode 100644 index 0000000000..1b6e9c9339 --- /dev/null +++ b/ceph-mon/templates/daemonset-mon.yaml @@ -0,0 +1,295 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "monLivenessProbeTemplate" -}} +exec: + command: + - /tmp/mon-check.sh +{{- end -}} + +{{- define "monReadinessProbeTemplate" -}} +exec: + command: + - /tmp/mon-check.sh +{{- end -}} + +{{- if and .Values.manifests.daemonset_mon .Values.deployment.ceph }} +{{- $envAll := . }} + +{{- $serviceAccountName := (printf "%s" .Release.Name) }} +{{ tuple $envAll "mon" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $serviceAccountName }} +rules: + - apiGroups: + - "" + resources: + - pods + - endpoints + verbs: + - get + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $serviceAccountName }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +{{- end }} + +{{- define "ceph.mon.daemonset" }} +{{- $daemonset := index . 0 }} +{{- $configMapName := index . 1 }} +{{- $serviceAccountName := index . 2 }} +{{- $envAll := index . 3 }} +{{- with $envAll }} +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: ceph-mon + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "ceph" "mon" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + selector: + matchLabels: +{{ tuple $envAll "ceph" "mon" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} +{{ tuple $envAll "mon" | include "helm-toolkit.snippets.kubernetes_upgrades_daemonset" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "ceph" "mon" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} +{{ dict "envAll" $envAll "podName" "ceph-mon" "containerNames" (list "ceph-mon" "ceph-init-dirs" "ceph-log-ownership") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "mon" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + nodeSelector: + {{ .Values.labels.mon.node_selector_key }}: {{ .Values.labels.mon.node_selector_value }} + hostNetwork: true + shareProcessNamespace: true + dnsPolicy: {{ .Values.pod.dns_policy }} + initContainers: +{{ tuple $envAll "mon" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + - name: ceph-init-dirs +{{ tuple $envAll "ceph_mon" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ dict "envAll" $envAll "application" "mon" "container" "ceph_init_dirs" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/init-dirs.sh + env: + - name: CLUSTER + value: "ceph" + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: pod-run + mountPath: /run + - name: pod-etc-ceph + mountPath: /etc/ceph + - name: ceph-mon-bin + mountPath: /tmp/init-dirs.sh + subPath: init-dirs.sh + readOnly: true + - name: pod-var-lib-ceph + mountPath: /var/lib/ceph + readOnly: false + - name: pod-var-lib-ceph-crash + mountPath: /var/lib/ceph/crash + readOnly: false + - name: ceph-log-ownership +{{ tuple $envAll "ceph_mon" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ dict "envAll" $envAll "application" "mon" "container" "ceph_log_ownership" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - chown + - -R + - ceph:root + - /var/log/ceph + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: pod-run + mountPath: /run + - name: pod-etc-ceph + mountPath: /etc/ceph + - name: pod-var-log + mountPath: /var/log/ceph + readOnly: false + containers: + - name: ceph-mon +{{ tuple $envAll "ceph_mon" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.mon | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "mon" "container" "ceph_mon" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + env: + - name: CLUSTER + value: "ceph" + - name: K8S_HOST_NETWORK + value: "1" + - name: MONMAP + value: /var/lib/ceph/mon/monmap + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: CEPH_PUBLIC_NETWORK + value: {{ .Values.network.public | quote }} + - name: KUBECTL_PARAM + value: {{ tuple $envAll "ceph" "mon" | include "helm-toolkit.snippets.kubernetes_kubectl_params" }} + - name: MON_PORT + value: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: MON_PORT_V2 + value: {{ tuple "ceph_mon" "internal" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: MON_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + command: + - /tmp/mon-start.sh + lifecycle: + preStop: + exec: + command: + - /tmp/mon-stop.sh + ports: + - containerPort: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - containerPort: {{ tuple "ceph_mon" "internal" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +{{ dict "envAll" . "component" "ceph" "container" "ceph-mon" "type" "liveness" "probeTemplate" (include "monLivenessProbeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | trim | indent 10 }} +{{ dict "envAll" . "component" "ceph" "container" "ceph-mon" "type" "readiness" "probeTemplate" (include "monReadinessProbeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | trim | indent 10 }} + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: pod-run + mountPath: /run + - name: pod-etc-ceph + mountPath: /etc/ceph + - name: ceph-mon-bin + mountPath: /tmp/mon-start.sh + subPath: mon-start.sh + readOnly: true + - name: ceph-mon-bin + mountPath: /tmp/mon-stop.sh + subPath: mon-stop.sh + readOnly: true + - name: ceph-mon-bin + mountPath: /tmp/mon-check.sh + subPath: mon-check.sh + readOnly: true + - name: ceph-mon-bin + mountPath: /tmp/checkObjectReplication.py + subPath: utils-checkObjectReplication.py + readOnly: true + - name: ceph-mon-bin + mountPath: /tmp/utils-checkDNS.sh + subPath: utils-checkDNS.sh + readOnly: true + - name: ceph-mon-etc + mountPath: /etc/ceph/ceph.conf.template + subPath: ceph.conf + readOnly: true + - name: ceph-client-admin-keyring + mountPath: /etc/ceph/ceph.client.admin.keyring + subPath: ceph.client.admin.keyring + readOnly: true + - name: ceph-mon-keyring + mountPath: /etc/ceph/ceph.mon.keyring.seed + subPath: ceph.mon.keyring + readOnly: true + - name: ceph-bootstrap-osd-keyring + mountPath: /var/lib/ceph/bootstrap-osd/ceph.keyring + subPath: ceph.keyring + readOnly: true + - name: ceph-bootstrap-mds-keyring + mountPath: /var/lib/ceph/bootstrap-mds/ceph.keyring + subPath: ceph.keyring + readOnly: true + - name: pod-var-lib-ceph + mountPath: /var/lib/ceph + readOnly: false + - name: pod-var-lib-ceph-crash + mountPath: /var/lib/ceph/crash + readOnly: false + - name: pod-var-log + mountPath: /var/log/ceph + readOnly: false + volumes: + - name: pod-tmp + emptyDir: {} + - name: pod-run + emptyDir: + medium: "Memory" + - name: pod-etc-ceph + emptyDir: {} + - name: pod-var-log + emptyDir: {} + - name: ceph-mon-bin + configMap: + name: {{ printf "%s-%s" $envAll.Release.Name "bin" | quote }} + defaultMode: 0555 + - name: ceph-mon-etc + configMap: + name: {{ $configMapName }} + defaultMode: 0444 + - name: pod-var-lib-ceph + hostPath: + path: {{ .Values.conf.storage.mon.directory }} + - name: pod-var-lib-ceph-crash + hostPath: + path: /var/lib/openstack-helm/ceph/crash + type: DirectoryOrCreate + - name: ceph-client-admin-keyring + secret: + secretName: {{ .Values.secrets.keyrings.admin }} + - name: ceph-mon-keyring + secret: + secretName: {{ .Values.secrets.keyrings.mon }} + - name: ceph-bootstrap-osd-keyring + secret: + secretName: {{ .Values.secrets.keyrings.osd }} + - name: ceph-bootstrap-mds-keyring + secret: + secretName: {{ .Values.secrets.keyrings.mds }} +{{- end }} +{{- end }} + +{{- if .Values.manifests.daemonset_mon }} +{{- $daemonset := .Values.daemonset.prefix_name }} +{{- $configMapName := (printf "%s-%s" .Release.Name "etc") }} +{{- $serviceAccountName := (printf "%s" .Release.Name) }} +{{- $daemonset_yaml := list $daemonset $configMapName $serviceAccountName . | include "ceph.mon.daemonset" | toString | fromYaml }} +{{- $configmap_yaml := "ceph.mon.configmap.etc" }} +{{- list $daemonset $daemonset_yaml $configmap_yaml $configMapName . | include "ceph.utils.mon_daemonset_overrides" }} +{{- end }} diff --git a/ceph-mon/templates/deployment-mgr.yaml b/ceph-mon/templates/deployment-mgr.yaml new file mode 100644 index 0000000000..7f2b4b1233 --- /dev/null +++ b/ceph-mon/templates/deployment-mgr.yaml @@ -0,0 +1,208 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "mgrLivenessProbeTemplate" -}} +exec: + command: + - /tmp/mgr-check.sh +{{- end }} + +{{- define "mgrReadinessProbeTemplate" -}} +exec: + command: + - /tmp/mgr-check.sh +{{- end }} + +{{- if and .Values.manifests.deployment_mgr (and .Values.deployment.ceph .Values.conf.features.mgr ) }} +{{- $envAll := . }} + +{{- $serviceAccountName := "ceph-mgr" }} +# This protective IF prevents an attempt of repeated creation +# of ceph-mgr service account. +# To be considered: the separation of SA and Deployment manifests. +{{- if .Values.manifests.deployment_mgr_sa }} +{{ tuple $envAll "mgr" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +{{- end }} +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: ceph-mgr + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "ceph" "mgr" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + replicas: {{ .Values.pod.replicas.mgr }} + selector: + matchLabels: +{{ tuple $envAll "ceph" "mgr" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} + strategy: + type: {{ .Values.pod.updateStrategy.mgr.type }} + template: + metadata: + labels: +{{ tuple $envAll "ceph" "mgr" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} +{{ dict "envAll" $envAll "podName" "ceph-mgr" "containerNames" (list "ceph-mgr" "ceph-init-dirs") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "mgr" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + affinity: +{{ tuple $envAll "ceph" "mgr" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} +{{ tuple $envAll "mgr" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }} + nodeSelector: + {{ .Values.labels.mgr.node_selector_key }}: {{ .Values.labels.mgr.node_selector_value }} + hostNetwork: true + hostPID: true + dnsPolicy: {{ .Values.pod.dns_policy }} + initContainers: +{{ tuple $envAll "mgr" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + - name: ceph-init-dirs +{{ tuple $envAll "ceph_mgr" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ dict "envAll" $envAll "application" "mgr" "container" "init_dirs" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/init-dirs.sh + env: + - name: CLUSTER + value: "ceph" + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: pod-run + mountPath: /run + - name: pod-etc-ceph + mountPath: /etc/ceph + - name: ceph-mon-bin + mountPath: /tmp/init-dirs.sh + subPath: init-dirs.sh + readOnly: true + - name: pod-var-lib-ceph + mountPath: /var/lib/ceph + readOnly: false + - name: pod-var-lib-ceph-crash + mountPath: /var/lib/ceph/crash + readOnly: false + containers: + - name: ceph-mgr +{{ tuple $envAll "ceph_mgr" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.mgr | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "mgr" "container" "mgr" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + env: + - name: CLUSTER + value: "ceph" + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: MON_PORT + value: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: MON_PORT_V2 + value: {{ tuple "ceph_mon" "internal" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + {{- if .Values.ceph_mgr_enabled_modules }} + - name: ENABLED_MODULES + value: |- + {{- range $value := .Values.ceph_mgr_enabled_modules }} + {{ $value }} + {{- end }} + {{- end }} + {{- if .Values.ceph_mgr_modules_config }} + {{- range $module,$params := .Values.ceph_mgr_modules_config }} + {{- range $key, $value := $params }} + - name: {{ $module }}_{{ $key }} + value: {{ $value | quote }} + {{- end }} + {{- end }} + {{- end }} + command: + - /mgr-start.sh + ports: + - name: mgr + containerPort: {{ tuple "ceph_mgr" "internal" "mgr" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + {{- if (has "prometheus" .Values.ceph_mgr_enabled_modules) }} + - name: metrics + containerPort: {{ tuple "ceph_mgr" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + {{ end -}} + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: pod-run + mountPath: /run + - name: pod-etc-ceph + mountPath: /etc/ceph + - name: ceph-mon-bin + mountPath: /mgr-start.sh + subPath: mgr-start.sh + readOnly: true + - name: ceph-mon-bin + mountPath: /tmp/mgr-check.sh + subPath: mgr-check.sh + readOnly: true + - name: ceph-mon-etc + mountPath: /etc/ceph/ceph.conf.template + subPath: ceph.conf + readOnly: true + - name: ceph-mon-admin-keyring + mountPath: /etc/ceph/ceph.client.admin.keyring + subPath: ceph.client.admin.keyring + readOnly: true + - name: ceph-bootstrap-mgr-keyring + mountPath: /var/lib/ceph/bootstrap-mgr/ceph.keyring + subPath: ceph.keyring + readOnly: false + - name: pod-var-lib-ceph + mountPath: /var/lib/ceph + readOnly: false + - name: pod-var-lib-ceph-crash + mountPath: /var/lib/ceph/crash + readOnly: false + - name: ceph-mon-bin + mountPath: /tmp/utils-checkPGs.py + subPath: utils-checkPGs.py + readOnly: true +{{ dict "envAll" . "component" "ceph" "container" "ceph-mgr" "type" "liveness" "probeTemplate" (include "mgrLivenessProbeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | trim | indent 10 }} +{{ dict "envAll" . "component" "ceph" "container" "ceph-mgr" "type" "readiness" "probeTemplate" (include "mgrReadinessProbeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | trim | indent 10 }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: pod-run + emptyDir: + medium: "Memory" + - name: pod-etc-ceph + emptyDir: {} + - name: ceph-mon-bin + configMap: + name: {{ printf "%s-%s" $envAll.Release.Name "bin" | quote }} + defaultMode: 0555 + - name: ceph-mon-etc + configMap: + name: {{ printf "%s-%s" $envAll.Release.Name "etc" | quote }} + defaultMode: 0444 + - name: pod-var-lib-ceph + emptyDir: {} + - name: pod-var-lib-ceph-crash + hostPath: + path: /var/lib/openstack-helm/ceph/crash + type: DirectoryOrCreate + - name: ceph-mon-admin-keyring + secret: + secretName: {{ .Values.secrets.keyrings.admin }} + - name: ceph-bootstrap-mgr-keyring + secret: + secretName: {{ .Values.secrets.keyrings.mgr }} +{{- end }} diff --git a/ceph-mon/templates/deployment-moncheck.yaml b/ceph-mon/templates/deployment-moncheck.yaml new file mode 100644 index 0000000000..1fb97a61cf --- /dev/null +++ b/ceph-mon/templates/deployment-moncheck.yaml @@ -0,0 +1,130 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.deployment_moncheck .Values.deployment.ceph }} +{{- $envAll := . }} + +{{- $serviceAccountName := "ceph-mon-check" }} +{{ tuple $envAll "moncheck" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: ceph-mon-check + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "ceph" "moncheck" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + replicas: {{ .Values.pod.replicas.mon_check }} + selector: + matchLabels: +{{ tuple $envAll "ceph" "moncheck" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} + template: + metadata: + labels: +{{ tuple $envAll "ceph" "moncheck" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} +{{ dict "envAll" $envAll "podName" "ceph-mon-check" "containerNames" (list "ceph-mon" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "moncheck" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + affinity: +{{ tuple $envAll "ceph" "moncheck" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} +{{ tuple $envAll "mon_check" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }} + nodeSelector: + {{ .Values.labels.mon.node_selector_key }}: {{ .Values.labels.mon.node_selector_value }} + initContainers: +{{ tuple $envAll "moncheck" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: ceph-mon +{{ tuple $envAll "ceph_mon_check" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.moncheck | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "moncheck" "container" "ceph_mon" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + env: + - name: CLUSTER + value: "ceph" + - name: K8S_HOST_NETWORK + value: "1" + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: MON_PORT + value: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: MON_PORT_V2 + value: {{ tuple "ceph_mon" "internal" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + command: + - /tmp/moncheck-start.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: pod-run + mountPath: /run + - name: pod-etc-ceph + mountPath: /etc/ceph + - name: ceph-mon-bin + mountPath: /tmp/moncheck-start.sh + subPath: moncheck-start.sh + readOnly: true + - name: ceph-mon-bin + mountPath: /tmp/moncheck-reap-zombies.py + subPath: moncheck-reap-zombies.py + readOnly: true + - name: ceph-mon-bin + mountPath: /tmp/utils-checkDNS.sh + subPath: utils-checkDNS.sh + readOnly: true + - name: ceph-mon-etc + mountPath: /etc/ceph/ceph.conf.template + subPath: ceph.conf + readOnly: true + - name: ceph-client-admin-keyring + mountPath: /etc/ceph/ceph.client.admin.keyring + subPath: ceph.client.admin.keyring + readOnly: true + - name: ceph-mon-keyring + mountPath: /etc/ceph/ceph.mon.keyring + subPath: ceph.mon.keyring + readOnly: true + - name: pod-var-lib-ceph + mountPath: /var/lib/ceph + readOnly: false + volumes: + - name: pod-tmp + emptyDir: {} + - name: pod-run + emptyDir: + medium: "Memory" + - name: pod-etc-ceph + emptyDir: {} + - name: ceph-mon-etc + configMap: + name: {{ printf "%s-%s" $envAll.Release.Name "etc" | quote }} + defaultMode: 0444 + - name: ceph-mon-bin + configMap: + name: {{ printf "%s-%s" $envAll.Release.Name "bin" | quote }} + defaultMode: 0555 + - name: pod-var-lib-ceph + emptyDir: {} + - name: ceph-client-admin-keyring + secret: + secretName: {{ .Values.secrets.keyrings.admin }} + - name: ceph-mon-keyring + secret: + secretName: {{ .Values.secrets.keyrings.mon }} +{{- end }} diff --git a/ceph-mon/templates/job-bootstrap.yaml b/ceph-mon/templates/job-bootstrap.yaml new file mode 100644 index 0000000000..1a4de7e906 --- /dev/null +++ b/ceph-mon/templates/job-bootstrap.yaml @@ -0,0 +1,85 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_bootstrap .Values.bootstrap.enabled }} +{{- $envAll := . }} + +{{- $serviceAccountName := "ceph-bootstrap" }} +{{ tuple $envAll "bootstrap" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: ceph-bootstrap + labels: +{{ tuple $envAll "ceph" "bootstrap" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +spec: + template: + metadata: + labels: +{{ tuple $envAll "ceph" "bootstrap" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} +{{ dict "envAll" $envAll "podName" "ceph-bootstrap" "containerNames" (list "ceph-bootstrap" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "bootstrap" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} + initContainers: +{{ tuple $envAll "bootstrap" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: ceph-bootstrap +{{ tuple $envAll "ceph_bootstrap" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.bootstrap | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "bootstrap" "container" "ceph_bootstrap" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/bootstrap.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: pod-etc-ceph + mountPath: /etc/ceph + - name: ceph-mon-bin + mountPath: /tmp/bootstrap.sh + subPath: bootstrap.sh + readOnly: true + - name: ceph-mon-etc + mountPath: /etc/ceph/ceph.conf + subPath: ceph.conf + readOnly: true + - name: ceph-client-admin-keyring + mountPath: /etc/ceph/ceph.client.admin.keyring + subPath: ceph.client.admin.keyring + readOnly: true + volumes: + - name: pod-tmp + emptyDir: {} + - name: pod-etc-ceph + emptyDir: {} + - name: ceph-mon-bin + configMap: + name: {{ printf "%s-%s" $envAll.Release.Name "bin" | quote }} + defaultMode: 0555 + - name: ceph-mon-etc + configMap: + name: {{ printf "%s-%s" $envAll.Release.Name "etc" | quote }} + defaultMode: 0444 + - name: ceph-client-admin-keyring + secret: + secretName: {{ .Values.secrets.keyrings.admin }} +{{- end }} diff --git a/ceph-mon/templates/job-image-repo-sync.yaml b/ceph-mon/templates/job-image-repo-sync.yaml new file mode 100644 index 0000000000..d378e43808 --- /dev/null +++ b/ceph-mon/templates/job-image-repo-sync.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "ceph-mon" -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/ceph-mon/templates/job-keyring.yaml b/ceph-mon/templates/job-keyring.yaml new file mode 100644 index 0000000000..112496dea1 --- /dev/null +++ b/ceph-mon/templates/job-keyring.yaml @@ -0,0 +1,133 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_keyring .Values.deployment.storage_secrets }} +{{- $envAll := . }} +{{- range $key1, $cephBootstrapKey := tuple "mds" "osd" "mon" "mgr" }} +{{- $component := print $cephBootstrapKey "-keyring-generator" }} +{{- $jobName := print "ceph-" $component }} + +{{- $serviceAccountName := $jobName }} +{{ tuple $envAll "job_keyring_generator" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $serviceAccountName }} +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $serviceAccountName }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ $jobName }} + labels: +{{ tuple $envAll "ceph" $jobName | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +spec: + template: + metadata: + labels: +{{ tuple $envAll "ceph" $jobName | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ dict "envAll" $envAll "podName" $jobName "containerNames" (list $jobName "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "ceph" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + nodeSelector: + {{ $envAll.Values.labels.job.node_selector_key }}: {{ $envAll.Values.labels.job.node_selector_value }} + initContainers: +{{ tuple $envAll "job_keyring_generator" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: {{ $jobName }} +{{ tuple $envAll "ceph_config_helper" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.secret_provisioning | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "ceph" "container" $jobName | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + env: + - name: DEPLOYMENT_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CEPH_GEN_DIR + value: /tmp + - name: CEPH_TEMPLATES_DIR + value: /tmp/templates + {{- if eq $cephBootstrapKey "mon" }} + - name: CEPH_KEYRING_NAME + value: ceph.mon.keyring + - name: CEPH_KEYRING_TEMPLATE + value: mon.keyring + {{- else }} + - name: CEPH_KEYRING_NAME + value: ceph.keyring + - name: CEPH_KEYRING_TEMPLATE + value: bootstrap.keyring.{{ $cephBootstrapKey }} + {{- end }} + - name: KUBE_SECRET_NAME + value: {{ index $envAll.Values.secrets.keyrings $cephBootstrapKey }} + command: + - /tmp/keys-bootstrap-keyring-manager.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: pod-etc-ceph + mountPath: /etc/ceph + - name: ceph-mon-bin + mountPath: /tmp/keys-bootstrap-keyring-manager.sh + subPath: keys-bootstrap-keyring-manager.sh + readOnly: true + - name: ceph-mon-bin + mountPath: /tmp/keys-bootstrap-keyring-generator.py + subPath: keys-bootstrap-keyring-generator.py + readOnly: true + - name: ceph-templates + mountPath: /tmp/templates + readOnly: true + volumes: + - name: pod-tmp + emptyDir: {} + - name: pod-etc-ceph + emptyDir: {} + - name: ceph-mon-bin + configMap: + name: {{ printf "%s-%s" $envAll.Release.Name "bin" | quote }} + defaultMode: 0555 + - name: ceph-templates + configMap: + name: {{ printf "%s-%s" $envAll.Release.Name "templates" | quote }} + defaultMode: 0444 +{{- end }} +{{- end }} diff --git a/ceph-mon/templates/job-post-apply.yaml b/ceph-mon/templates/job-post-apply.yaml new file mode 100644 index 0000000000..0b924cc42f --- /dev/null +++ b/ceph-mon/templates/job-post-apply.yaml @@ -0,0 +1,145 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if eq .Values.pod.lifecycle.upgrades.daemonsets.pod_replacement_strategy "OnDelete" }} +{{- if and .Values.manifests.job_post_apply }} +{{- $envAll := . }} + +{{- $serviceAccountName := printf "%s-%s" .Release.Name "post-apply" }} +{{ tuple $envAll "post-apply" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ $serviceAccountName }} +rules: + - apiGroups: + - '' + resources: + - pods + - events + - jobs + - pods/exec + verbs: + - create + - get + - delete + - list + - apiGroups: + - 'apps' + resources: + - daemonsets + verbs: + - get + - list + - apiGroups: + - 'batch' + resources: + - jobs + verbs: + - get + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ $serviceAccountName }} + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ $serviceAccountName }} + labels: +{{ tuple $envAll "ceph-upgrade" "post-apply" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +spec: + template: + metadata: + labels: +{{ tuple $envAll "ceph-upgrade" "post-apply" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} +{{ dict "envAll" $envAll "podName" "ceph-mon-post-apply" "containerNames" (list "ceph-mon-post-apply" "init" ) | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "post_apply" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} + initContainers: +{{ tuple $envAll "post-apply" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: ceph-mon-post-apply +{{ tuple $envAll "ceph_config_helper" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.bootstrap | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "post_apply" "container" "ceph_mon_post_apply" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + env: + - name: CLUSTER + value: "ceph" + - name: CEPH_NAMESPACE + value: {{ .Release.Namespace }} + - name: RELEASE_GROUP_NAME + value: {{ .Release.Name }} + - name: UNCONDITIONAL_MON_RESTART + value: {{ .Values.conf.storage.unconditional_mon_restart | quote }} + command: + - /tmp/post-apply.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: pod-etc-ceph + mountPath: /etc/ceph + - name: ceph-mon-bin + mountPath: /tmp/post-apply.sh + subPath: post-apply.sh + readOnly: true + - name: ceph-mon-bin + mountPath: /tmp/wait-for-pods.sh + subPath: wait-for-pods.sh + readOnly: true + - name: ceph-mon-etc + mountPath: /etc/ceph/ceph.conf + subPath: ceph.conf + readOnly: true + - name: ceph-mon-admin-keyring + mountPath: /etc/ceph/ceph.client.admin.keyring + subPath: ceph.client.admin.keyring + readOnly: true + volumes: + - name: pod-tmp + emptyDir: {} + - name: pod-etc-ceph + emptyDir: {} + - name: ceph-mon-bin + configMap: + name: {{ printf "%s-%s" $envAll.Release.Name "bin" | quote }} + defaultMode: 0555 + - name: ceph-mon-etc + configMap: + name: {{ printf "%s-%s" $envAll.Release.Name "etc" | quote }} + defaultMode: 0444 + - name: ceph-mon-admin-keyring + secret: + secretName: {{ .Values.secrets.keyrings.admin }} +{{- end }} +{{- end }} diff --git a/ceph-mon/templates/job-storage-admin-keys.yaml b/ceph-mon/templates/job-storage-admin-keys.yaml new file mode 100644 index 0000000000..0456f54e16 --- /dev/null +++ b/ceph-mon/templates/job-storage-admin-keys.yaml @@ -0,0 +1,132 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_storage_admin_keys .Values.deployment.storage_secrets }} +{{- $envAll := . }} + +{{- $serviceAccountName := "ceph-storage-keys-generator" }} +{{ tuple $envAll "storage_keys_generator" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $serviceAccountName }} +rules: + - apiGroups: + - "" + resources: + - pods + - pods/exec + - secrets + verbs: + - get + - create + - patch + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $serviceAccountName }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: ceph-storage-keys-generator + labels: +{{ tuple $envAll "ceph" "storage-keys-generator" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +spec: + template: + metadata: + labels: +{{ tuple $envAll "ceph" "storage-keys-generator" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} +{{ dict "envAll" $envAll "podName" "ceph-storage-keys-generator" "containerNames" (list "ceph-storage-keys-generator" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "storage_keys_generator" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + nodeSelector: + {{ $envAll.Values.labels.job.node_selector_key }}: {{ $envAll.Values.labels.job.node_selector_value }} + initContainers: +{{ tuple $envAll "storage_keys_generator" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: ceph-storage-keys-generator +{{ tuple $envAll "ceph_config_helper" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.secret_provisioning | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "storage_keys_generator" "container" "ceph_storage_keys_generator" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + env: + - name: DEPLOYMENT_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: CEPH_GEN_DIR + value: /tmp + - name: CEPH_TEMPLATES_DIR + value: /tmp/templates + - name: CEPH_KEYRING_NAME + value: ceph.client.admin.keyring + - name: CEPH_KEYRING_TEMPLATE + value: admin.keyring + - name: CEPH_KEYRING_ADMIN_NAME + value: {{ .Values.secrets.keyrings.admin }} + - name: CEPH_STORAGECLASS_ADMIN_SECRET_NAME + value: {{ .Values.storageclass.rbd.parameters.adminSecretName }} + - name: CEPH_STORAGECLASS_ADMIN_SECRET_NAME_NODE + value: {{ .Values.storageclass.rbd.parameters.adminSecretNameNode }} + command: + - /tmp/keys-storage-keyring-manager.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: pod-etc-ceph + mountPath: /etc/ceph + - name: ceph-mon-bin + mountPath: /tmp/keys-storage-keyring-manager.sh + subPath: keys-storage-keyring-manager.sh + readOnly: true + - name: ceph-mon-bin + mountPath: /tmp/keys-bootstrap-keyring-generator.py + subPath: keys-bootstrap-keyring-generator.py + readOnly: true + - name: ceph-templates + mountPath: /tmp/templates + readOnly: true + volumes: + - name: pod-tmp + emptyDir: {} + - name: pod-etc-ceph + emptyDir: {} + - name: ceph-mon-bin + configMap: + name: {{ printf "%s-%s" $envAll.Release.Name "bin" | quote }} + defaultMode: 0555 + - name: ceph-templates + configMap: + name: {{ printf "%s-%s" $envAll.Release.Name "templates" | quote }} + defaultMode: 0444 +{{- end }} diff --git a/ceph-mon/templates/secret-registry.yaml b/ceph-mon/templates/secret-registry.yaml new file mode 100644 index 0000000000..da979b3223 --- /dev/null +++ b/ceph-mon/templates/secret-registry.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_registry .Values.endpoints.oci_image_registry.auth.enabled }} +{{ include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) }} +{{- end }} diff --git a/ceph-mon/templates/service-mgr.yaml b/ceph-mon/templates/service-mgr.yaml new file mode 100644 index 0000000000..bef61141c2 --- /dev/null +++ b/ceph-mon/templates/service-mgr.yaml @@ -0,0 +1,42 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.service_mgr ( and .Values.deployment.ceph .Values.conf.features.mgr ) }} +{{- $envAll := . }} +{{- $prometheus_annotations := $envAll.Values.monitoring.prometheus.ceph_mgr }} +--- +apiVersion: v1 +kind: Service +metadata: + name: ceph-mgr + labels: +{{ tuple $envAll "ceph" "manager" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: +{{- if .Values.monitoring.prometheus.enabled }} +{{ tuple $prometheus_annotations | include "helm-toolkit.snippets.prometheus_service_annotations" | indent 4 }} +{{- end }} +spec: + ports: + - name: ceph-mgr + port: {{ tuple "ceph_mgr" "internal" "mgr" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + protocol: TCP + targetPort: {{ tuple "ceph_mgr" "internal" "mgr" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + {{ if (has "prometheus" .Values.ceph_mgr_enabled_modules) }} + - name: metrics + protocol: TCP + port: {{ tuple "ceph_mgr" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + {{ end }} + selector: +{{ tuple $envAll "ceph" "mgr" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- end }} diff --git a/ceph-mon/templates/service-mon-discovery.yaml b/ceph-mon/templates/service-mon-discovery.yaml new file mode 100644 index 0000000000..04582ff7e5 --- /dev/null +++ b/ceph-mon/templates/service-mon-discovery.yaml @@ -0,0 +1,41 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.service_mon_discovery .Values.deployment.ceph }} +{{- $envAll := . }} +--- +kind: Service +apiVersion: v1 +metadata: + name: {{ tuple "ceph_mon" "discovery" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +spec: + ports: + - name: mon + port: {{ tuple "ceph_mon" "discovery" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + protocol: TCP + targetPort: {{ tuple "ceph_mon" "discovery" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - name: mon-msgr2 + port: {{ tuple "ceph_mon" "discovery" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + protocol: TCP + targetPort: {{ tuple "ceph_mon" "discovery" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + selector: +{{- if .Values.manifests.daemonset_mon }} +{{ tuple $envAll "ceph" "mon" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- else }} + app: rook-ceph-mon + ceph_daemon_type: mon +{{- end }} + clusterIP: None + publishNotReadyAddresses: true +{{- end }} diff --git a/ceph-mon/templates/service-mon.yaml b/ceph-mon/templates/service-mon.yaml new file mode 100644 index 0000000000..3ef29df5d2 --- /dev/null +++ b/ceph-mon/templates/service-mon.yaml @@ -0,0 +1,35 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.service_mon .Values.deployment.ceph }} +{{- $envAll := . }} +--- +kind: Service +apiVersion: v1 +metadata: + name: {{ tuple "ceph_mon" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +spec: + ports: + - name: mon + port: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + protocol: TCP + targetPort: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - name: mon-msgr2 + port: {{ tuple "ceph_mon" "internal" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + protocol: TCP + targetPort: {{ tuple "ceph_mon" "internal" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + selector: +{{ tuple $envAll "ceph" "mon" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + clusterIP: None +{{- end }} diff --git a/ceph-mon/templates/snippets/_mon_host_from_k8s_ep.sh.tpl b/ceph-mon/templates/snippets/_mon_host_from_k8s_ep.sh.tpl new file mode 100644 index 0000000000..eb71898251 --- /dev/null +++ b/ceph-mon/templates/snippets/_mon_host_from_k8s_ep.sh.tpl @@ -0,0 +1,68 @@ +{{- define "ceph-mon.snippets.mon_host_from_k8s_ep" -}} +{{/* + +Inserts a bash function definition mon_host_from_k8s_ep() which can be used +to construct a mon_hosts value from the given namespaced endpoint. + +Usage (e.g. in _script.sh.tpl): + #!/bin/bash + + : "${NS:=ceph}" + : "${EP:=ceph-mon-discovery}" + + {{ include "ceph-mon.snippets.mon_host_from_k8s_ep" . }} + + MON_HOST=$(mon_host_from_k8s_ep "$NS" "$EP") + + if [ -z "$MON_HOST" ]; then + # deal with failure + else + sed -i -e "s/^mon_host = /mon_host = $MON_HOST/" /etc/ceph/ceph.conf + fi +*/}} +{{` +# Construct a mon_hosts value from the given namespaced endpoint +# IP x.x.x.x with port p named "mon-msgr2" will appear as [v2:x.x.x.x/p/0] +# IP x.x.x.x with port q named "mon" will appear as [v1:x.x.x.x/q/0] +# IP x.x.x.x with ports p and q will appear as [v2:x.x.x.x/p/0,v1:x.x.x.x/q/0] +# The entries for all IPs will be joined with commas +mon_host_from_k8s_ep() { + local ns=$1 + local ep=$2 + + if [ -z "$ns" ] || [ -z "$ep" ]; then + return 1 + fi + + # We don't want shell expansion for the go-template expression + # shellcheck disable=SC2016 + kubectl get endpoints -n "$ns" "$ep" -o go-template=' + {{- $sep := "" }} + {{- range $_,$s := .subsets }} + {{- $v2port := 0 }} + {{- $v1port := 0 }} + {{- range $_,$port := index $s "ports" }} + {{- if (eq $port.name "mon-msgr2") }} + {{- $v2port = $port.port }} + {{- else if (eq $port.name "mon") }} + {{- $v1port = $port.port }} + {{- end }} + {{- end }} + {{- range $_,$address := index $s "addresses" }} + {{- $v2endpoint := printf "v2:%s:%d/0" $address.ip $v2port }} + {{- $v1endpoint := printf "v1:%s:%d/0" $address.ip $v1port }} + {{- if (and $v2port $v1port) }} + {{- printf "%s[%s,%s]" $sep $v2endpoint $v1endpoint }} + {{- $sep = "," }} + {{- else if $v2port }} + {{- printf "%s[%s]" $sep $v2endpoint }} + {{- $sep = "," }} + {{- else if $v1port }} + {{- printf "%s[%s]" $sep $v1endpoint }} + {{- $sep = "," }} + {{- end }} + {{- end }} + {{- end }}' +} +`}} +{{- end -}} diff --git a/ceph-mon/templates/utils/_mon_daemonset_overrides.tpl b/ceph-mon/templates/utils/_mon_daemonset_overrides.tpl new file mode 100644 index 0000000000..ac1998e745 --- /dev/null +++ b/ceph-mon/templates/utils/_mon_daemonset_overrides.tpl @@ -0,0 +1,287 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "ceph.utils.match_exprs_hash" }} + {{- $match_exprs := index . 0 }} + {{- $context := index . 1 }} + {{- $_ := set $context.Values "__match_exprs_hash_content" "" }} + {{- range $match_expr := $match_exprs }} + {{- $_ := set $context.Values "__match_exprs_hash_content" (print $context.Values.__match_exprs_hash_content $match_expr.key $match_expr.operator ($match_expr.values | quote)) }} + {{- end }} + {{- $context.Values.__match_exprs_hash_content | sha256sum | trunc 8 }} + {{- $_ := unset $context.Values "__match_exprs_hash_content" }} +{{- end }} + +{{- define "ceph.utils.mon_daemonset_overrides" }} + {{- $daemonset := index . 0 }} + {{- $daemonset_yaml := index . 1 }} + {{- $configmap_include := index . 2 }} + {{- $configmap_name := index . 3 }} + {{- $context := index . 4 }} + {{- $_ := unset $context ".Files" }} + {{- $_ := set $context.Values "__daemonset_yaml" $daemonset_yaml }} + {{- $daemonset_root_name := printf "ceph_%s" $daemonset }} + {{- $_ := set $context.Values "__daemonset_list" list }} + {{- $_ := set $context.Values "__default" dict }} + {{- if hasKey $context.Values.conf "overrides" }} + {{- range $key, $val := $context.Values.conf.overrides }} + + {{- if eq $key $daemonset_root_name }} + {{- range $type, $type_data := . }} + + {{- if eq $type "hosts" }} + {{- range $host_data := . }} + {{/* dictionary that will contain all info needed to generate this + iteration of the daemonset */}} + {{- $current_dict := dict }} + + {{/* set daemonset name */}} + {{- $_ := set $current_dict "name" $host_data.name }} + + {{/* apply overrides */}} + {{- $override_conf_copy := $host_data.conf }} + {{/* Deep copy to prevent https://storyboard.openstack.org/#!/story/2005936 */}} + {{- $root_conf_copy := omit ($context.Values.conf | toYaml | fromYaml) "overrides" }} + {{- $merged_dict := mergeOverwrite $root_conf_copy $override_conf_copy }} + {{- $root_conf_copy2 := dict "conf" $merged_dict }} + {{- $context_values := omit (omit ($context.Values | toYaml | fromYaml) "conf") "__daemonset_list" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2 }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $current_dict "nodeData" $root_conf_copy4 }} + + {{/* Schedule to this host explicitly. */}} + {{- $nodeSelector_dict := dict }} + + {{- $_ := set $nodeSelector_dict "key" "kubernetes.io/hostname" }} + {{- $_ := set $nodeSelector_dict "operator" "In" }} + + {{- $values_list := list $host_data.name }} + {{- $_ := set $nodeSelector_dict "values" $values_list }} + + {{- $list_aggregate := list $nodeSelector_dict }} + {{- $_ := set $current_dict "matchExpressions" $list_aggregate }} + + {{/* store completed daemonset entry/info into global list */}} + {{- $list_aggregate := append $context.Values.__daemonset_list $current_dict }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + + {{- end }} + {{- end }} + + {{- if eq $type "labels" }} + {{- $_ := set $context.Values "__label_list" . }} + {{- range $label_data := . }} + {{/* dictionary that will contain all info needed to generate this + iteration of the daemonset. */}} + {{- $_ := set $context.Values "__current_label" dict }} + + {{/* set daemonset name */}} + {{- $_ := set $context.Values.__current_label "name" $label_data.label.key }} + + {{/* apply overrides */}} + {{- $override_conf_copy := $label_data.conf }} + {{/* Deep copy to prevent https://storyboard.openstack.org/#!/story/2005936 */}} + {{- $root_conf_copy := omit ($context.Values.conf | toYaml | fromYaml) "overrides" }} + {{- $merged_dict := mergeOverwrite $root_conf_copy $override_conf_copy }} + {{- $root_conf_copy2 := dict "conf" $merged_dict }} + {{- $context_values := omit (omit ($context.Values | toYaml | fromYaml) "conf") "__daemonset_list" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2 }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__current_label "nodeData" $root_conf_copy4 }} + + {{/* Schedule to the provided label value(s) */}} + {{- $label_dict := omit $label_data.label "NULL" }} + {{- $_ := set $label_dict "operator" "In" }} + {{- $list_aggregate := list $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + + {{/* Do not schedule to other specified labels, with higher + precedence as the list position increases. Last defined label + is highest priority. */}} + {{- $other_labels := without $context.Values.__label_list $label_data }} + {{- range $label_data2 := $other_labels }} + {{- $label_dict := omit $label_data2.label "NULL" }} + + {{- $_ := set $label_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__current_label.matchExpressions $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + {{- end }} + {{- $_ := set $context.Values "__label_list" $other_labels }} + + {{/* Do not schedule to any other specified hosts */}} + {{- range $type, $type_data := $val }} + {{- if eq $type "hosts" }} + {{- range $host_data := . }} + {{- $label_dict := dict }} + + {{- $_ := set $label_dict "key" "kubernetes.io/hostname" }} + {{- $_ := set $label_dict "operator" "NotIn" }} + + {{- $values_list := list $host_data.name }} + {{- $_ := set $label_dict "values" $values_list }} + + {{- $list_aggregate := append $context.Values.__current_label.matchExpressions $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + {{- end }} + {{- end }} + {{- end }} + + {{/* store completed daemonset entry/info into global list */}} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__current_label }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + {{- $_ := unset $context.Values "__current_label" }} + + {{- end }} + {{- end }} + {{- end }} + + {{/* scheduler exceptions for the default daemonset */}} + {{- $_ := set $context.Values.__default "matchExpressions" list }} + + {{- range $type, $type_data := . }} + {{/* Do not schedule to other specified labels */}} + {{- if eq $type "labels" }} + {{- range $label_data := . }} + {{- $default_dict := omit $label_data.label "NULL" }} + + {{- $_ := set $default_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__default.matchExpressions $default_dict }} + {{- $_ := set $context.Values.__default "matchExpressions" $list_aggregate }} + {{- end }} + {{- end }} + {{/* Do not schedule to other specified hosts */}} + {{- if eq $type "hosts" }} + {{- range $host_data := . }} + {{- $default_dict := dict }} + + {{- $_ := set $default_dict "key" "kubernetes.io/hostname" }} + {{- $_ := set $default_dict "operator" "NotIn" }} + + {{- $values_list := list $host_data.name }} + {{- $_ := set $default_dict "values" $values_list }} + + {{- $list_aggregate := append $context.Values.__default.matchExpressions $default_dict }} + {{- $_ := set $context.Values.__default "matchExpressions" $list_aggregate }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + + {{/* generate the default daemonset */}} + + {{/* set name */}} + {{- $_ := set $context.Values.__default "name" "default" }} + + {{/* no overrides apply, so copy as-is */}} + {{- $root_conf_copy1 := omit $context.Values.conf "overrides" }} + {{- $root_conf_copy2 := dict "conf" $root_conf_copy1 }} + {{- $context_values := omit $context.Values "conf" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2 }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__default "nodeData" $root_conf_copy4 }} + + {{/* add to global list */}} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__default }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + + {{- $_ := set $context.Values "__last_configmap_name" $configmap_name }} + {{- range $current_dict := $context.Values.__daemonset_list }} + + {{- $context_novalues := omit $context "Values" }} + {{- $merged_dict := mergeOverwrite $context_novalues $current_dict.nodeData }} + {{- $_ := set $current_dict "nodeData" $merged_dict }} + + {{/* name needs to be a DNS-1123 compliant name. Ensure lower case */}} + {{- $name_format1 := printf (print $daemonset_root_name "-" $current_dict.name) | lower }} + {{/* labels may contain underscores which would be invalid here, so we replace them with dashes + there may be other valid label names which would make for an invalid DNS-1123 name + but these will be easier to handle in future with sprig regex* functions + (not availabile in helm 2.5.1) */}} + {{- $name_format2 := $name_format1 | replace "_" "-" | replace "." "-" }} + {{/* To account for the case where the same label is defined multiple times in overrides + (but with different label values), we add a sha of the scheduling data to ensure + name uniqueness */}} + {{- $_ := set $current_dict "dns_1123_name" dict }} + {{- if hasKey $current_dict "matchExpressions" }} + {{- $_ := set $current_dict "dns_1123_name" (printf (print $name_format2 "-" (list $current_dict.matchExpressions $context | include "ceph.utils.match_exprs_hash"))) }} + {{- else }} + {{- $_ := set $current_dict "dns_1123_name" $name_format2 }} + {{- end }} + + {{/* set daemonset metadata name */}} + {{- if not $context.Values.__daemonset_yaml.metadata }}{{- $_ := set $context.Values.__daemonset_yaml "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.metadata.name }}{{- $_ := set $context.Values.__daemonset_yaml.metadata "name" dict }}{{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.metadata "name" $current_dict.dns_1123_name }} + + {{/* cross-reference configmap name to container volume definitions */}} + {{- $_ := set $context.Values "__volume_list" list }} + {{- range $current_volume := $context.Values.__daemonset_yaml.spec.template.spec.volumes }} + {{- $_ := set $context.Values "__volume" $current_volume }} + {{- if hasKey $context.Values.__volume "configMap" }} + {{- if eq $context.Values.__volume.configMap.name $context.Values.__last_configmap_name }} + {{- $_ := set $context.Values.__volume.configMap "name" $current_dict.dns_1123_name }} + {{- end }} + {{- end }} + {{- $updated_list := append $context.Values.__volume_list $context.Values.__volume }} + {{- $_ := set $context.Values "__volume_list" $updated_list }} + {{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "volumes" $context.Values.__volume_list }} + + {{/* populate scheduling restrictions */}} + {{- if hasKey $current_dict "matchExpressions" }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "affinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity "nodeAffinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity "requiredDuringSchedulingIgnoredDuringExecution" dict }}{{- end }} + {{- $match_exprs := dict }} + {{- $_ := set $match_exprs "matchExpressions" $current_dict.matchExpressions }} + {{- $appended_match_expr := list $match_exprs }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" $appended_match_expr }} + {{- end }} + + {{/* input value hash for current set of values overrides */}} + {{- if not $context.Values.__daemonset_yaml.spec }}{{- $_ := set $context.Values.__daemonset_yaml "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template }}{{- $_ := set $context.Values.__daemonset_yaml.spec "template" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata.annotations }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata "annotations" dict }}{{- end }} + {{- $cmap := list $current_dict.dns_1123_name $current_dict.nodeData | include $configmap_include }} + {{- $values_hash := $cmap | quote | sha256sum }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-etc-hash" $values_hash }} + + {{/* generate configmap */}} +--- +{{ $cmap }} + + {{/* generate daemonset yaml */}} +{{ range $k, $v := index $current_dict.nodeData.Values.conf.storage "mon" }} +--- +{{- $_ := set $context.Values "__tmpYAML" dict }} + +{{ $dsNodeName := index $context.Values.__daemonset_yaml.metadata "name" }} +{{ $localDsNodeName := print (trunc 54 $current_dict.dns_1123_name) "-" (print $dsNodeName $k | quote | sha256sum | trunc 8) }} +{{- if not $context.Values.__tmpYAML.metadata }}{{- $_ := set $context.Values.__tmpYAML "metadata" dict }}{{- end }} +{{- $_ := set $context.Values.__tmpYAML.metadata "name" $localDsNodeName }} + +{{ merge $context.Values.__tmpYAML $context.Values.__daemonset_yaml | toYaml }} + +{{ end }} + +--- + {{- $_ := set $context.Values "__last_configmap_name" $current_dict.dns_1123_name }} + {{- end }} +{{- end }} diff --git a/ceph-mon/values.yaml b/ceph-mon/values.yaml new file mode 100644 index 0000000000..ac65353633 --- /dev/null +++ b/ceph-mon/values.yaml @@ -0,0 +1,530 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for ceph-mon. +# This is a YAML-formatted file. +# Declare name/value pairs to be passed into your templates. +# name: value + +--- +deployment: + ceph: true + storage_secrets: true + +images: + pull_policy: IfNotPresent + tags: + ceph_bootstrap: 'docker.io/openstackhelm/ceph-daemon:ubuntu_jammy_19.2.1-1-20250207' + ceph_config_helper: 'docker.io/openstackhelm/ceph-config-helper:ubuntu_jammy_19.2.1-1-20250207' + ceph_mon: 'docker.io/openstackhelm/ceph-daemon:ubuntu_jammy_19.2.1-1-20250207' + ceph_mgr: 'docker.io/openstackhelm/ceph-daemon:ubuntu_jammy_19.2.1-1-20250207' + ceph_mon_check: 'docker.io/openstackhelm/ceph-config-helper:ubuntu_jammy_19.2.1-1-20250207' + dep_check: 'quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal' + image_repo_sync: 'docker.io/library/docker:17.07.0' + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +labels: + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + mon: + node_selector_key: ceph-mon + node_selector_value: enabled + mgr: + node_selector_key: ceph-mgr + node_selector_value: enabled + +pod: + security_context: + mon: + pod: + runAsUser: 65534 + container: + ceph_init_dirs: + runAsUser: 0 + readOnlyRootFilesystem: true + ceph_log_ownership: + runAsUser: 0 + readOnlyRootFilesystem: true + ceph_mon: + runAsUser: 64045 + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + mgr: + pod: + runAsUser: 65534 + container: + init_dirs: + runAsUser: 0 + readOnlyRootFilesystem: true + mgr: + runAsUser: 64045 + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + moncheck: + pod: + runAsUser: 65534 + container: + ceph_mon: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + bootstrap: + pod: + runAsUser: 65534 + container: + ceph_bootstrap: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + storage_keys_generator: + pod: + runAsUser: 65534 + container: + ceph_storage_keys_generator: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + ceph: + pod: + runAsUser: 65534 + container: + ceph-mds-keyring-generator: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + ceph-mgr-keyring-generator: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + ceph-mon-keyring-generator: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + ceph-osd-keyring-generator: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + post_apply: + pod: + runAsUser: 65534 + container: + ceph_mon_post_apply: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + dns_policy: "ClusterFirstWithHostNet" + replicas: + mgr: 2 + mon_check: 1 + lifecycle: + upgrades: + daemonsets: + pod_replacement_strategy: RollingUpdate + mon: + enabled: true + min_ready_seconds: 0 + max_unavailable: 1 + updateStrategy: + mgr: + type: Recreate + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + weight: + default: 10 + resources: + enabled: false + mon: + requests: + memory: "50Mi" + cpu: "250m" + limits: + memory: "100Mi" + cpu: "500m" + mgr: + requests: + memory: "5Mi" + cpu: "250m" + limits: + memory: "50Mi" + cpu: "500m" + mon_check: + requests: + memory: "5Mi" + cpu: "250m" + limits: + memory: "50Mi" + cpu: "500m" + jobs: + bootstrap: + limits: + memory: "1024Mi" + cpu: "2000m" + requests: + memory: "128Mi" + cpu: "500m" + secret_provisioning: + limits: + memory: "1024Mi" + cpu: "2000m" + requests: + memory: "128Mi" + cpu: "500m" + image_repo_sync: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + tolerations: + mgr: + tolerations: + - effect: NoExecute + key: node.kubernetes.io/not-ready + operator: Exists + tolerationSeconds: 60 + - effect: NoExecute + key: node.kubernetes.io/unreachable + operator: Exists + tolerationSeconds: 60 + mon_check: + tolerations: + - effect: NoExecute + key: node.kubernetes.io/not-ready + operator: Exists + tolerationSeconds: 60 + - effect: NoExecute + key: node.kubernetes.io/unreachable + operator: Exists + tolerationSeconds: 60 + probes: + ceph: + ceph-mon: + readiness: + enabled: true + params: + initialDelaySeconds: 60 + periodSeconds: 60 + timeoutSeconds: 5 + liveness: + enabled: true + params: + initialDelaySeconds: 360 + periodSeconds: 180 + timeoutSeconds: 5 + ceph-mgr: + readiness: + enabled: true + params: + initialDelaySeconds: 30 + timeoutSeconds: 5 + liveness: + enabled: true + params: + initialDelaySeconds: 30 + timeoutSeconds: 5 + +secrets: + keyrings: + mon: ceph-mon-keyring + mds: ceph-bootstrap-mds-keyring + osd: ceph-bootstrap-osd-keyring + mgr: ceph-bootstrap-mgr-keyring + admin: ceph-client-admin-keyring + oci_image_registry: + ceph-mon: ceph-mon-oci-image-registry-key + +network: + public: 192.168.0.0/16 + cluster: 192.168.0.0/16 + +conf: + features: + mgr: true + templates: + keyring: + admin: | + [client.admin] + key = {{ key }} + auid = 0 + caps mds = "allow" + caps mon = "allow *" + caps osd = "allow *" + caps mgr = "allow *" + mon: | + [mon.] + key = {{ key }} + caps mon = "allow *" + bootstrap: + mds: | + [client.bootstrap-mds] + key = {{ key }} + caps mon = "allow profile bootstrap-mds" + mgr: | + [client.bootstrap-mgr] + key = {{ key }} + caps mgr = "allow profile bootstrap-mgr" + osd: | + [client.bootstrap-osd] + key = {{ key }} + caps mon = "allow profile bootstrap-osd" + ceph: + global: + # auth + cephx: true + cephx_require_signatures: false + cephx_cluster_require_signatures: true + cephx_service_require_signatures: false + objecter_inflight_op_bytes: "1073741824" + objecter_inflight_ops: 10240 + debug_ms: "0/0" + mon_osd_down_out_interval: 1800 + mon_osd_down_out_subtree_limit: root + mon_osd_min_in_ratio: 0 + mon_osd_min_up_ratio: 0 + mon_data_avail_warn: 15 + log_file: /dev/stdout + mon_cluster_log_file: /dev/stdout + # Beginning with the Pacific release, this config setting is necessary + # to allow pools to use 1x replication, which is disabled by default. The + # openstack-helm gate scripts use 1x replication for automated testing, + # so this is required. It doesn't seem to be sufficient to add this to + # /etc/ceph/ceph.conf, however. It must also be set explicitly via the + # 'ceph config' command, so this must also be added to the + # cluster_commands value in the ceph-client chart so it will be set + # before pools are created and configured there. + mon_allow_pool_size_one: true + osd: + osd_mkfs_type: xfs + osd_mkfs_options_xfs: -f -i size=2048 + osd_max_object_name_len: 256 + ms_bind_port_min: 6800 + ms_bind_port_max: 7100 + osd_snap_trim_priority: 1 + osd_snap_trim_sleep: 0.1 + osd_pg_max_concurrent_snap_trims: 1 + filestore_merge_threshold: -10 + filestore_split_multiple: 12 + filestore_max_sync_interval: 10 + osd_scrub_begin_hour: 22 + osd_scrub_end_hour: 4 + osd_scrub_during_recovery: false + osd_scrub_sleep: 0.1 + osd_scrub_chunk_min: 1 + osd_scrub_chunk_max: 4 + osd_scrub_load_threshold: 10.0 + osd_deep_scrub_stride: "1048576" + osd_scrub_priority: 1 + osd_recovery_op_priority: 1 + osd_recovery_max_active: 1 + osd_mount_options_xfs: "rw,noatime,largeio,inode64,swalloc,logbufs=8,logbsize=256k,allocsize=4M" + osd_journal_size: 10240 + storage: + mon: + directory: /var/lib/openstack-helm/ceph/mon + + # The post-apply job will try to determine if mons need to be restarted + # and only restart them if necessary. Set this value to "true" to restart + # mons unconditionally. + unconditional_mon_restart: "false" + +daemonset: + prefix_name: "mon" + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - ceph-mon-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + bootstrap: + jobs: null + services: + - endpoint: internal + service: ceph_mon + job_keyring_generator: + jobs: null + mon: + jobs: + - ceph-storage-keys-generator + - ceph-mon-keyring-generator + mgr: + jobs: + - ceph-storage-keys-generator + - ceph-mgr-keyring-generator + services: + - endpoint: internal + service: ceph_mon + moncheck: + jobs: + - ceph-storage-keys-generator + - ceph-mon-keyring-generator + services: + - endpoint: discovery + service: ceph_mon + storage_keys_generator: + jobs: null + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + +bootstrap: + enabled: false + script: | + ceph -s + function ensure_pool () { + ceph osd pool stats $1 || ceph osd pool create $1 $2 + if [[ $(ceph mon versions | awk '/version/{print $3}' | cut -d. -f1) -ge 12 ]]; then + ceph osd pool application enable $1 $3 + fi + } + #ensure_pool volumes 8 cinder + +# Uncomment below to enable mgr modules +# For a list of available modules: +# http://docs.ceph.com/docs/master/mgr/ +# This overrides mgr_initial_modules (default: restful, status) +# Any module not listed here will be disabled +ceph_mgr_enabled_modules: + - restful + - status + - prometheus + - balancer + - iostat + - pg_autoscaler + +# You can configure your mgr modules +# below. Each module has its own set +# of key/value. Refer to the doc +# above for more info. For example: +ceph_mgr_modules_config: +# balancer: +# active: 1 +# prometheus: + # server_port: 9283 +# server_addr: 0.0.0.0 +# dashboard: +# port: 7000 +# localpool: +# failure_domain: host +# subtree: rack +# pg_num: "128" +# num_rep: "3" +# min_size: "2" + +# if you change provision_storage_class to false +# it is presumed you manage your own storage +# class definition externally +# We iterate over each storageclass parameters +# and derive the manifest. +storageclass: + rbd: + parameters: + adminSecretName: pvc-ceph-conf-combined-storageclass + adminSecretNameNode: pvc-ceph-conf-combined-storageclass + cephfs: + provision_storage_class: true + provisioner: ceph.com/cephfs + metadata: + name: cephfs + parameters: + adminId: admin + userSecretName: pvc-ceph-cephfs-client-key + adminSecretName: pvc-ceph-conf-combined-storageclass + adminSecretNamespace: ceph + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + ceph-mon: + username: ceph-mon + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + ceph_mon: + namespace: null + hosts: + default: ceph-mon + discovery: ceph-mon-discovery + host_fqdn_override: + default: null + port: + mon: + default: 6789 + mon_msgr2: + default: 3300 + ceph_mgr: + namespace: null + hosts: + default: ceph-mgr + host_fqdn_override: + default: null + port: + mgr: + default: 7000 + metrics: + default: 9283 + scheme: + default: http + +monitoring: + prometheus: + enabled: true + ceph_mgr: + scrape: true + port: 9283 + +manifests: + configmap_bin: true + configmap_etc: true + configmap_templates: true + daemonset_mon: true + deployment_mgr: true + deployment_mgr_sa: true + deployment_moncheck: true + job_image_repo_sync: true + job_bootstrap: true + job_keyring: true + job_post_apply: true + service_mon: true + service_mgr: true + service_mon_discovery: true + job_storage_admin_keys: true + secret_registry: true +... diff --git a/ceph-osd/Chart.yaml b/ceph-osd/Chart.yaml new file mode 100644 index 0000000000..6bb0b54b43 --- /dev/null +++ b/ceph-osd/Chart.yaml @@ -0,0 +1,24 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: v1.0.0 +description: OpenStack-Helm Ceph OSD +name: ceph-osd +version: 2024.2.0 +home: https://github.com/ceph/ceph +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit + version: ">= 0.1.0" +... diff --git a/ceph-osd/templates/bin/_bootstrap.sh.tpl b/ceph-osd/templates/bin/_bootstrap.sh.tpl new file mode 100644 index 0000000000..6452d0a073 --- /dev/null +++ b/ceph-osd/templates/bin/_bootstrap.sh.tpl @@ -0,0 +1,18 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +{{ .Values.bootstrap.script | default "echo 'Not Enabled'" }} diff --git a/ceph-osd/templates/bin/_helm-tests.sh.tpl b/ceph-osd/templates/bin/_helm-tests.sh.tpl new file mode 100644 index 0000000000..9008ad8816 --- /dev/null +++ b/ceph-osd/templates/bin/_helm-tests.sh.tpl @@ -0,0 +1,64 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +function check_osd_count() { + echo "#### Start: Checking OSD count ####" + noup_flag=$(ceph osd stat | awk '/noup/ {print $2}') + osd_stat=$(ceph osd stat -f json-pretty) + num_osd=$(awk '/"num_osds"/{print $2}' <<< "$osd_stat" | cut -d, -f1) + num_in_osds=$(awk '/"num_in_osds"/{print $2}' <<< "$osd_stat" | cut -d, -f1) + num_up_osds=$(awk '/"num_up_osds"/{print $2}' <<< "$osd_stat" | cut -d, -f1) + + MIN_OSDS=$((${num_osd}*$REQUIRED_PERCENT_OF_OSDS/100)) + if [ ${MIN_OSDS} -lt 1 ]; then + MIN_OSDS=1 + fi + + if [ "${noup_flag}" ]; then + osd_status=$(ceph osd dump -f json | jq -c '.osds[] | .state') + count=0 + for osd in $osd_status; do + if [[ "$osd" == *"up"* || "$osd" == *"new"* ]]; then + ((count=count+1)) + fi + done + echo "Caution: noup flag is set. ${count} OSDs in up/new state. Required number of OSDs: ${MIN_OSDS}." + ceph -s + exit 0 + else + if [ "${num_osd}" -eq 0 ]; then + echo "There are no osds in the cluster" + elif [ "${num_in_osds}" -ge "${MIN_OSDS}" ] && [ "${num_up_osds}" -ge "${MIN_OSDS}" ]; then + echo "Required number of OSDs (${MIN_OSDS}) are UP and IN status" + ceph -s + exit 0 + else + echo "Required number of OSDs (${MIN_OSDS}) are NOT UP and IN status. Cluster shows OSD count=${num_osd}, UP=${num_up_osds}, IN=${num_in_osds}" + fi + fi +} + +# in case the chart has been re-installed in order to make changes to daemonset +# we do not need rack_by_rack restarts +# but we need to wait until all re-installed ceph-osd pods are healthy +# and there is degraded objects +while true; do + check_osd_count + sleep 60 +done + diff --git a/ceph-osd/templates/bin/_init-dirs.sh.tpl b/ceph-osd/templates/bin/_init-dirs.sh.tpl new file mode 100644 index 0000000000..03f8c39650 --- /dev/null +++ b/ceph-osd/templates/bin/_init-dirs.sh.tpl @@ -0,0 +1,32 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +export LC_ALL=C +: "${OSD_BOOTSTRAP_KEYRING:=/var/lib/ceph/bootstrap-osd/${CLUSTER}.keyring}" + +mkdir -p "$(dirname "${OSD_BOOTSTRAP_KEYRING}")" + +# Let's create the ceph directories +for DIRECTORY in osd tmp crash; do + mkdir -p "/var/lib/ceph/${DIRECTORY}" +done + +# Create socket directory +mkdir -p /run/ceph + +# Adjust the owner of all those directories +chown -R ceph. /run/ceph/ /var/lib/ceph/* diff --git a/ceph-osd/templates/bin/_post-apply.sh.tpl b/ceph-osd/templates/bin/_post-apply.sh.tpl new file mode 100644 index 0000000000..c2fe97a167 --- /dev/null +++ b/ceph-osd/templates/bin/_post-apply.sh.tpl @@ -0,0 +1,227 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +export LC_ALL=C + +: "${ADMIN_KEYRING:=/etc/ceph/${CLUSTER}.client.admin.keyring}" + +if [[ ! -f /etc/ceph/${CLUSTER}.conf ]]; then + echo "ERROR- /etc/ceph/${CLUSTER}.conf must exist; get it from your existing mon" + exit 1 +fi + +if [[ ! -f ${ADMIN_KEYRING} ]]; then + echo "ERROR- ${ADMIN_KEYRING} must exist; get it from your existing mon" + exit 1 +fi + +ceph --cluster ${CLUSTER} -s +function wait_for_pods() { + timeout=${2:-1800} + end=$(date -ud "${timeout} seconds" +%s) + # Selecting containers with "ceph-osd-default" name and + # counting them based on "ready" field. + count_pods=".items | map(.status.containerStatuses | .[] | \ + select(.name==\"ceph-osd-default\")) | \ + group_by(.ready) | map({(.[0].ready | tostring): length}) | .[]" + min_osds="add | if .true >= (.false + .true)*${REQUIRED_PERCENT_OF_OSDS}/100 \ + then \"pass\" else \"fail\" end" + while true; do + # Leaving while loop if minimum amount of OSDs are ready. + # It allows to proceed even if some OSDs are not ready + # or in "CrashLoopBackOff" state + state=$(kubectl get pods --namespace="${1}" -l component=osd -o json | jq "${count_pods}") + osd_state=$(jq -s "${min_osds}" <<< "${state}") + if [[ "${osd_state}" == \"pass\" ]]; then + break + fi + sleep 5 + + if [ $(date -u +%s) -gt $end ] ; then + echo -e "Containers failed to start after $timeout seconds\n" + kubectl get pods --namespace "${1}" -o wide -l component=osd + exit 1 + fi + done +} + +function check_ds() { + for ds in `kubectl get ds --namespace=$CEPH_NAMESPACE -l component=osd --no-headers=true|awk '{print $1}'` + do + ds_query=`kubectl get ds -n $CEPH_NAMESPACE $ds -o json|jq -r .status` + if echo $ds_query |grep -i "numberAvailable" ;then + currentNumberScheduled=`echo $ds_query|jq -r .currentNumberScheduled` + desiredNumberScheduled=`echo $ds_query|jq -r .desiredNumberScheduled` + numberAvailable=`echo $ds_query|jq -r .numberAvailable` + numberReady=`echo $ds_query|jq -r .numberReady` + updatedNumberScheduled=`echo $ds_query|jq -r .updatedNumberScheduled` + ds_check=`echo "$currentNumberScheduled $desiredNumberScheduled $numberAvailable $numberReady $updatedNumberScheduled"| \ + tr ' ' '\n'|sort -u|wc -l` + if [ $ds_check != 1 ]; then + echo "few pods under daemonset $ds are not yet ready" + exit + else + echo "all pods ubder deamonset $ds are ready" + fi + else + echo "this are no osds under daemonset $ds" + fi + done +} + +function wait_for_pgs () { + echo "#### Start: Checking pgs ####" + + pgs_ready=0 + pgs_inactive=0 + query='map({state: .state}) | group_by(.state) | map({state: .[0].state, count: length}) | .[] | select(.state | contains("active") | not)' + + if [[ $(ceph mon versions | awk '/version/{print $3}' | cut -d. -f1) -ge 14 ]]; then + query=".pg_stats | ${query}" + fi + + # Loop until all pgs are active + while [[ $pgs_ready -lt 3 ]]; do + pgs_state=$(ceph --cluster ${CLUSTER} pg ls -f json | jq -c "${query}") + if [[ $(jq -c '. | select(.state | contains("peering") | not)' <<< "${pgs_state}") ]]; then + if [[ $pgs_inactive -gt 200 ]]; then + # If inactive PGs aren't peering after ~10 minutes, fail + echo "Failure, found inactive PGs that aren't peering" + exit 1 + fi + (( pgs_inactive+=1 )) + else + pgs_inactive=0 + fi + if [[ "${pgs_state}" ]]; then + pgs_ready=0 + else + (( pgs_ready+=1 )) + fi + sleep 30 + done +} + +function wait_for_degraded_objects () { + echo "#### Start: Checking for degraded objects ####" + + # Loop until no degraded objects + while [[ ! -z "`ceph --cluster ${CLUSTER} -s | grep 'degraded'`" ]] + do + sleep 30 + ceph -s + done +} + +function wait_for_degraded_and_misplaced_objects () { + echo "#### Start: Checking for degraded and misplaced objects ####" + + # Loop until no degraded or misplaced objects + while [[ ! -z "`ceph --cluster ${CLUSTER} -s | grep 'degraded\|misplaced'`" ]] + do + sleep 30 + ceph -s + done +} + +function restart_by_rack() { + + racks=`ceph osd tree | awk '/rack/{print $4}'` + echo "Racks under ceph cluster are: $racks" + for rack in $racks + do + hosts_in_rack=(`ceph osd tree | sed -n "/rack $rack/,/rack/p" | awk '/host/{print $4}' | tr '\n' ' '|sed 's/ *$//g'`) + echo "hosts under rack "$rack" are: ${hosts_in_rack[@]}" + echo "hosts count under $rack are: ${#hosts_in_rack[@]}" + for host in ${hosts_in_rack[@]} + do + echo "host is : $host" + if [[ ! -z "$host" ]]; then + pods_on_host=$(kubectl get po -n "$CEPH_NAMESPACE" -l component=osd -o wide |grep "$host"|awk '{print $1}' | tr '\n' ' '|sed 's/ *$//g') + echo "Restarting the pods under host $host" + for pod in ${pods_on_host} + do + kubectl delete pod -n "$CEPH_NAMESPACE" "${pod}" || true + done + fi + done + echo "waiting for the pods under host $host from restart" + # The pods will not be ready in first 60 seconds. Thus we can reduce + # amount of queries to kubernetes. + sleep 60 + # Degraded objects won't recover with noout set unless pods come back and + # PGs become healthy, so simply wait for 0 degraded objects + wait_for_degraded_objects + ceph -s + done +} + +if [[ "$DISRUPTIVE_OSD_RESTART" != "true" ]]; then + wait_for_pods $CEPH_NAMESPACE +fi + +require_upgrade=0 +max_release=0 + +for ds in `kubectl get ds --namespace=$CEPH_NAMESPACE -l component=osd --no-headers=true|awk '{print $1}'` +do + updatedNumberScheduled=`kubectl get ds -n $CEPH_NAMESPACE $ds -o json|jq -r .status.updatedNumberScheduled` + desiredNumberScheduled=`kubectl get ds -n $CEPH_NAMESPACE $ds -o json|jq -r .status.desiredNumberScheduled` + if [[ $updatedNumberScheduled != $desiredNumberScheduled ]]; then + if kubectl get ds -n $CEPH_NAMESPACE $ds -o json|jq -r .status|grep -i "numberAvailable" ;then + require_upgrade=$((require_upgrade+1)) + _release=`kubectl get ds -n $CEPH_NAMESPACE $ds -o json|jq -r .status.observedGeneration` + max_release=$(( max_release > _release ? max_release : _release )) + fi + fi +done + +echo "Latest revision of the helm chart(s) is : $max_release" + +# If flags are set that will prevent recovery, don't restart OSDs +ceph -s | grep "noup\|noin\|nobackfill\|norebalance\|norecover" > /dev/null +if [[ $? -ne 0 ]]; then + if [[ "$UNCONDITIONAL_OSD_RESTART" == "true" ]] || [[ $max_release -gt 1 ]]; then + if [[ "$UNCONDITIONAL_OSD_RESTART" == "true" ]] || [[ $require_upgrade -gt 0 ]]; then + if [[ "$DISRUPTIVE_OSD_RESTART" == "true" ]]; then + echo "restarting all osds simultaneously" + kubectl -n $CEPH_NAMESPACE delete pod -l component=osd + sleep 60 + echo "waiting for pgs to become active and for degraded objects to recover" + wait_for_pgs + wait_for_degraded_objects + ceph -s + else + echo "waiting for inactive pgs and degraded objects before upgrade" + wait_for_pgs + wait_for_degraded_and_misplaced_objects + ceph -s + ceph osd "set" noout + echo "lets restart the osds rack by rack" + restart_by_rack + ceph osd "unset" noout + fi + fi + + #lets check all the ceph-osd daemonsets + echo "checking DS" + check_ds + else + echo "No revisions found for upgrade" + fi +else + echo "Skipping OSD restarts because flags are set that would prevent recovery" +fi diff --git a/ceph-osd/templates/bin/osd/_check.sh.tpl b/ceph-osd/templates/bin/osd/_check.sh.tpl new file mode 100644 index 0000000000..3ed90d01a2 --- /dev/null +++ b/ceph-osd/templates/bin/osd/_check.sh.tpl @@ -0,0 +1,41 @@ +#!/bin/sh + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# A liveness check for ceph OSDs: exit 0 if +# all OSDs on this host are in the "active" state +# per their admin sockets. + +SOCKDIR=${CEPH_SOCKET_DIR:-/run/ceph} +SBASE=${CEPH_OSD_SOCKET_BASE:-ceph-osd} +SSUFFIX=${CEPH_SOCKET_SUFFIX:-asok} + +# default: no sockets, not live +cond=1 +for sock in $SOCKDIR/$SBASE.*.$SSUFFIX; do + if [ -S $sock ]; then + OSD_ID=$(echo $sock | awk -F. '{print $2}') + OSD_STATE=$(ceph -f json --connect-timeout 1 --admin-daemon "${sock}" status|jq -r '.state') + echo "OSD ${OSD_ID} ${OSD_STATE}"; + # Succeed if the OSD state is active (running) or preboot (starting) + if [ "${OSD_STATE}" = "active" ] || [ "${OSD_STATE}" = "preboot" ]; then + cond=0 + else + # Any other state is unexpected and the probe fails + exit 1 + fi + else + echo "No daemon sockets found in $SOCKDIR" + fi +done +exit $cond diff --git a/ceph-osd/templates/bin/osd/_directory.sh.tpl b/ceph-osd/templates/bin/osd/_directory.sh.tpl new file mode 100644 index 0000000000..e32342730d --- /dev/null +++ b/ceph-osd/templates/bin/osd/_directory.sh.tpl @@ -0,0 +1,106 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +export LC_ALL=C + +source /tmp/osd-common-ceph-volume.sh + +: "${JOURNAL_DIR:=/var/lib/ceph/journal}" + +if [[ ! -d /var/lib/ceph/osd ]]; then + echo "ERROR- could not find the osd directory, did you bind mount the OSD data directory?" + echo "ERROR- use -v :/var/lib/ceph/osd" + exit 1 +fi + +# check if anything is present, if not, create an osd and its directory +if [[ -n "$(find /var/lib/ceph/osd -type d -empty ! -name "lost+found")" ]]; then + echo "Creating osd" + UUID=$(uuidgen) + OSD_SECRET=$(ceph-authtool --gen-print-key) + OSD_ID=$(echo "{\"cephx_secret\": \"${OSD_SECRET}\"}" | ceph osd new ${UUID} -i - -n client.bootstrap-osd -k "$OSD_BOOTSTRAP_KEYRING") + + # test that the OSD_ID is an integer + if [[ "$OSD_ID" =~ ^-?[0-9]+$ ]]; then + echo "OSD created with ID: ${OSD_ID}" + else + echo "OSD creation failed: ${OSD_ID}" + exit 1 + fi + + OSD_PATH="$OSD_PATH_BASE-$OSD_ID/" + if [ -n "${JOURNAL_DIR}" ]; then + OSD_JOURNAL="${JOURNAL_DIR}/journal.${OSD_ID}" + chown -R ceph. ${JOURNAL_DIR} + else + if [ -n "${JOURNAL}" ]; then + OSD_JOURNAL=${JOURNAL} + chown -R ceph. $(dirname ${JOURNAL_DIR}) + else + OSD_JOURNAL=${OSD_PATH%/}/journal + fi + fi + # create the folder and own it + mkdir -p "${OSD_PATH}" + echo "created folder ${OSD_PATH}" + # write the secret to the osd keyring file + ceph-authtool --create-keyring ${OSD_PATH%/}/keyring --name osd.${OSD_ID} --add-key ${OSD_SECRET} + chown -R "${CHOWN_OPT[@]}" ceph. "${OSD_PATH}" + OSD_KEYRING="${OSD_PATH%/}/keyring" + # init data directory + ceph-osd -i ${OSD_ID} --mkfs --osd-uuid ${UUID} --mkjournal --osd-journal ${OSD_JOURNAL} --setuser ceph --setgroup ceph + # add the osd to the crush map + crush_location +fi + +for OSD_ID in $(ls /var/lib/ceph/osd | sed 's/.*-//'); do + # NOTE(gagehugo): Writing the OSD_ID to tmp for logging + echo "${OSD_ID}" > /tmp/osd-id + OSD_PATH="$OSD_PATH_BASE-$OSD_ID/" + OSD_KEYRING="${OSD_PATH%/}/keyring" + if [ -n "${JOURNAL_DIR}" ]; then + OSD_JOURNAL="${JOURNAL_DIR}/journal.${OSD_ID}" + chown -R ceph. ${JOURNAL_DIR} + else + if [ -n "${JOURNAL}" ]; then + OSD_JOURNAL=${JOURNAL} + chown -R ceph. $(dirname ${JOURNAL_DIR}) + else + OSD_JOURNAL=${OSD_PATH%/}/journal + chown ceph. ${OSD_JOURNAL} + fi + fi + # log osd filesystem type + FS_TYPE=`stat --file-system -c "%T" ${OSD_PATH}` + echo "OSD $OSD_PATH filesystem type: $FS_TYPE" + + # NOTE(supamatt): Just in case permissions do not align up, we recursively set them correctly. + if [ $(stat -c%U ${OSD_PATH}) != ceph ]; then + chown -R ceph. ${OSD_PATH}; + fi + + crush_location +done + +exec /usr/bin/ceph-osd \ + --cluster ${CLUSTER} \ + -f \ + -i ${OSD_ID} \ + --osd-journal ${OSD_JOURNAL} \ + -k ${OSD_KEYRING} + --setuser ceph \ + --setgroup disk $! > /run/ceph-osd.pid diff --git a/ceph-osd/templates/bin/osd/_init.sh.tpl b/ceph-osd/templates/bin/osd/_init.sh.tpl new file mode 100644 index 0000000000..2f74d2df37 --- /dev/null +++ b/ceph-osd/templates/bin/osd/_init.sh.tpl @@ -0,0 +1,20 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +echo "Initializing the osd with ${DEPLOY_TOOL}" +exec "/tmp/init-${DEPLOY_TOOL}.sh" diff --git a/ceph-osd/templates/bin/osd/_log-runner-stop.sh.tpl b/ceph-osd/templates/bin/osd/_log-runner-stop.sh.tpl new file mode 100644 index 0000000000..4658c9855c --- /dev/null +++ b/ceph-osd/templates/bin/osd/_log-runner-stop.sh.tpl @@ -0,0 +1,35 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +source /tmp/utils-resolveLocations.sh + +touch /tmp/ceph-log-runner.stop + +TAIL_PID="$(cat /tmp/ceph-log-runner.pid)" +while kill -0 ${TAIL_PID} >/dev/null 2>&1; +do + kill -9 ${TAIL_PID}; + sleep 1; +done + +SLEEP_PID="$(cat /tmp/ceph-log-runner-sleep.pid)" +while kill -0 ${SLEEP_PID} >/dev/null 2>&1; +do + kill -9 ${SLEEP_PID}; + sleep 1; +done diff --git a/ceph-osd/templates/bin/osd/_log-tail.sh.tpl b/ceph-osd/templates/bin/osd/_log-tail.sh.tpl new file mode 100644 index 0000000000..e903760fee --- /dev/null +++ b/ceph-osd/templates/bin/osd/_log-tail.sh.tpl @@ -0,0 +1,53 @@ +#!/bin/bash +set -ex + +osd_id_file="/tmp/osd-id" + +function wait_for_file() { + local file="$1"; shift + local wait_seconds="${1:-30}"; shift + + until test $((wait_seconds--)) -eq 0 -o -f "$file" ; do + sleep 1 + done + + ((++wait_seconds)) +} +wait_for_file "${osd_id_file}" "${WAIT_FOR_OSD_ID_TIMEOUT}" + +log_file="/var/log/ceph/${DAEMON_NAME}.$(cat "${osd_id_file}").log" +wait_for_file "${log_file}" "${WAIT_FOR_OSD_ID_TIMEOUT}" + +trap "exit" SIGTERM SIGINT +keep_running=true + +function tail_file () { + while $keep_running; do + tail --retry -f "${log_file}" & + tail_pid=$! + echo $tail_pid > /tmp/ceph-log-runner-tail.pid + wait $tail_pid + if [ -f /tmp/ceph-log-runner.stop ]; then + keep_running=false + fi + sleep 30 + done +} + +function truncate_log () { + while $keep_running; do + sleep ${TRUNCATE_PERIOD} + sleep_pid=$! + echo $sleep_pid > /tmp/ceph-log-runner-sleep.pid + if [[ -f ${log_file} ]] ; then + truncate -s "${TRUNCATE_SIZE}" "${log_file}" + fi + done +} + +tail_file & +truncate_log & + +wait -n +keep_running=false +wait diff --git a/ceph-osd/templates/bin/osd/_start.sh.tpl b/ceph-osd/templates/bin/osd/_start.sh.tpl new file mode 100644 index 0000000000..c8ff581303 --- /dev/null +++ b/ceph-osd/templates/bin/osd/_start.sh.tpl @@ -0,0 +1,20 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +echo "LAUNCHING OSD: in ${STORAGE_TYPE%-*}:${STORAGE_TYPE#*-} mode" +exec "/tmp/osd-${STORAGE_TYPE%-*}-${DEPLOY_TOOL}.sh" diff --git a/ceph-osd/templates/bin/osd/_stop.sh.tpl b/ceph-osd/templates/bin/osd/_stop.sh.tpl new file mode 100644 index 0000000000..fdb2dda00d --- /dev/null +++ b/ceph-osd/templates/bin/osd/_stop.sh.tpl @@ -0,0 +1,35 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +source /tmp/utils-resolveLocations.sh + +CEPH_OSD_PID="$(cat /run/ceph-osd.pid)" +while kill -0 ${CEPH_OSD_PID} >/dev/null 2>&1; do + kill -SIGTERM ${CEPH_OSD_PID} + sleep 1 +done + +if [ "x${STORAGE_TYPE%-*}" == "xblock" ]; then + OSD_DEVICE=$(readlink -f ${STORAGE_LOCATION}) + OSD_JOURNAL=$(readlink -f ${JOURNAL_LOCATION}) + if [ "x${STORAGE_TYPE#*-}" == "xlogical" ]; then + umount "$(findmnt -S "${OSD_DEVICE}1" | tail -n +2 | awk '{ print $1 }')" + fi +fi + +fi diff --git a/ceph-osd/templates/bin/osd/ceph-volume/_block.sh.tpl b/ceph-osd/templates/bin/osd/ceph-volume/_block.sh.tpl new file mode 100644 index 0000000000..987aa2801d --- /dev/null +++ b/ceph-osd/templates/bin/osd/ceph-volume/_block.sh.tpl @@ -0,0 +1,142 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +source /tmp/osd-common-ceph-volume.sh + +set -ex + +: "${OSD_SOFT_FORCE_ZAP:=1}" +: "${OSD_JOURNAL_DISK:=}" + +if [ "x${STORAGE_TYPE%-*}" == "xdirectory" ]; then + export OSD_DEVICE="/var/lib/ceph/osd" +else + export OSD_DEVICE=$(readlink -f ${STORAGE_LOCATION}) +fi + +if [ "x$JOURNAL_TYPE" == "xdirectory" ]; then + export OSD_JOURNAL="/var/lib/ceph/journal" +else + export OSD_JOURNAL=$(readlink -f ${JOURNAL_LOCATION}) +fi + +if [[ -z "${OSD_DEVICE}" ]];then + echo "ERROR- You must provide a device to build your OSD ie: /dev/sdb" + exit 1 +fi + +if [[ ! -b "${OSD_DEVICE}" ]]; then + echo "ERROR- The device pointed by OSD_DEVICE ${OSD_DEVICE} doesn't exist !" + exit 1 +fi + +ACTIVATE_OPTIONS="" +CEPH_OSD_OPTIONS="" + +udev_settle + +OSD_ID=$(ceph-volume inventory ${OSD_DEVICE} | grep "osd id" | awk '{print $3}') +if [[ -z ${OSD_ID} ]]; then + echo "OSD_ID not found from device ${OSD_DEVICE}" + exit 1 +fi +OSD_FSID=$(ceph-volume inventory ${OSD_DEVICE} | grep "osd fsid" | awk '{print $3}') +if [[ -z ${OSD_FSID} ]]; then + echo "OSD_FSID not found from device ${OSD_DEVICE}" + exit 1 +fi +OSD_PATH="${OSD_PATH_BASE}-${OSD_ID}" +OSD_KEYRING="${OSD_PATH}/keyring" + +mkdir -p ${OSD_PATH} + +ceph-volume lvm -v \ + --setuser ceph \ + --setgroup disk \ + activate ${ACTIVATE_OPTIONS} \ + --auto-detect-objectstore \ + --no-systemd ${OSD_ID} ${OSD_FSID} + +# NOTE(stevetaylor): Set the OSD's crush weight (use noin flag to prevent rebalancing if necessary) +OSD_WEIGHT=$(get_osd_crush_weight_from_device ${OSD_DEVICE}) +# NOTE(supamatt): add or move the OSD's CRUSH location +crush_location + +if [ "${OSD_BLUESTORE:-0}" -ne 1 ]; then + if [ -n "${OSD_JOURNAL}" ]; then + if [ -b "${OSD_JOURNAL}" ]; then + OSD_JOURNAL_DISK="$(readlink -f ${OSD_PATH}/journal)" + if [ -z "${OSD_JOURNAL_DISK}" ]; then + echo "ERROR: Unable to find journal device ${OSD_JOURNAL_DISK}" + exit 1 + else + OSD_JOURNAL="${OSD_JOURNAL_DISK}" + if [ -e "${OSD_PATH}/run_mkjournal" ]; then + ceph-osd -i ${OSD_ID} --mkjournal + rm -rf ${OSD_PATH}/run_mkjournal + fi + fi + fi + if [ "x${JOURNAL_TYPE}" == "xdirectory" ]; then + OSD_JOURNAL="${OSD_JOURNAL}/journal.${OSD_ID}" + touch ${OSD_JOURNAL} + wait_for_file "${OSD_JOURNAL}" + else + if [ ! -b "${OSD_JOURNAL}" ]; then + echo "ERROR: Unable to find journal device ${OSD_JOURNAL}" + exit 1 + else + chown ceph. "${OSD_JOURNAL}" + fi + fi + else + wait_for_file "${OSD_JOURNAL}" + chown ceph. "${OSD_JOURNAL}" + fi +fi + +# NOTE(supamatt): Just in case permissions do not align up, we recursively set them correctly. +if [ $(stat -c%U ${OSD_PATH}) != ceph ]; then + chown -R ceph. ${OSD_PATH}; +fi + +# NOTE(gagehugo): Writing the OSD_ID to tmp for logging +echo "${OSD_ID}" > /tmp/osd-id + +if [ "x${JOURNAL_TYPE}" == "xdirectory" ]; then + chown -R ceph. /var/lib/ceph/journal + ceph-osd \ + --cluster ceph \ + --osd-data ${OSD_PATH} \ + --osd-journal ${OSD_JOURNAL} \ + -f \ + -i ${OSD_ID} \ + --setuser ceph \ + --setgroup disk \ + --mkjournal +fi + +exec /usr/bin/ceph-osd \ + --cluster ${CLUSTER} \ + ${CEPH_OSD_OPTIONS} \ + -f \ + -i ${OSD_ID} \ + --setuser ceph \ + --setgroup disk & echo $! > /run/ceph-osd.pid +wait + +# Clean up resources held by the common script +common_cleanup diff --git a/ceph-osd/templates/bin/osd/ceph-volume/_bluestore.sh.tpl b/ceph-osd/templates/bin/osd/ceph-volume/_bluestore.sh.tpl new file mode 100644 index 0000000000..a74c8a8e93 --- /dev/null +++ b/ceph-osd/templates/bin/osd/ceph-volume/_bluestore.sh.tpl @@ -0,0 +1,103 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +source /tmp/osd-common-ceph-volume.sh + +set -ex + +: "${OSD_SOFT_FORCE_ZAP:=1}" + +export OSD_DEVICE=$(readlink -f ${STORAGE_LOCATION}) + +if [[ -z "${OSD_DEVICE}" ]];then + echo "ERROR- You must provide a device to build your OSD ie: /dev/sdb" + exit 1 +fi + +if [[ ! -b "${OSD_DEVICE}" ]]; then + echo "ERROR- The device pointed by OSD_DEVICE ${OSD_DEVICE} doesn't exist !" + exit 1 +fi + +ACTIVATE_OPTIONS="" +CEPH_OSD_OPTIONS="" + +udev_settle + +OSD_ID=$(get_osd_id_from_device ${OSD_DEVICE}) +if [[ -z ${OSD_ID} ]]; then + echo "OSD_ID not found from device ${OSD_DEVICE}" + exit 1 +fi +OSD_FSID=$(get_osd_fsid_from_device ${OSD_DEVICE}) +if [[ -z ${OSD_FSID} ]]; then + echo "OSD_FSID not found from device ${OSD_DEVICE}" + exit 1 +fi +OSD_PATH="${OSD_PATH_BASE}-${OSD_ID}" +OSD_KEYRING="${OSD_PATH}/keyring" + +mkdir -p ${OSD_PATH} + +ceph-volume lvm -v \ + --setuser ceph \ + --setgroup disk \ + activate ${ACTIVATE_OPTIONS} \ + --auto-detect-objectstore \ + --no-systemd ${OSD_ID} ${OSD_FSID} +# Cross check the db and wal symlinks if missed +DB_DEV=$(get_osd_db_device_from_device ${OSD_DEVICE}) +if [[ ! -z ${DB_DEV} ]]; then + if [[ ! -h /var/lib/ceph/osd/ceph-${OSD_ID}/block.db ]]; then + ln -snf ${DB_DEV} /var/lib/ceph/osd/ceph-${OSD_ID}/block.db + chown -h ceph:ceph ${DB_DEV} + chown -h ceph:ceph /var/lib/ceph/osd/ceph-${OSD_ID}/block.db + fi +fi +WAL_DEV=$(get_osd_wal_device_from_device ${OSD_DEVICE}) +if [[ ! -z ${WAL_DEV} ]]; then + if [[ ! -h /var/lib/ceph/osd/ceph-${OSD_ID}/block.wal ]]; then + ln -snf ${WAL_DEV} /var/lib/ceph/osd/ceph-${OSD_ID}/block.wal + chown -h ceph:ceph ${WAL_DEV} + chown -h ceph:ceph /var/lib/ceph/osd/ceph-${OSD_ID}/block.wal + fi +fi + +# NOTE(stevetaylor): Set the OSD's crush weight (use noin flag to prevent rebalancing if necessary) +OSD_WEIGHT=$(get_osd_crush_weight_from_device ${OSD_DEVICE}) +# NOTE(supamatt): add or move the OSD's CRUSH location +crush_location + + +# NOTE(supamatt): Just in case permissions do not align up, we recursively set them correctly. +if [ $(stat -c%U ${OSD_PATH}) != ceph ]; then + chown -R ceph. ${OSD_PATH}; +fi + +# NOTE(gagehugo): Writing the OSD_ID to tmp for logging +echo "${OSD_ID}" > /tmp/osd-id + +exec /usr/bin/ceph-osd \ + --cluster ${CLUSTER} \ + ${CEPH_OSD_OPTIONS} \ + -f \ + -i ${OSD_ID} \ + --setuser ceph \ + --setgroup disk & echo $! > /run/ceph-osd.pid +wait + +# Clean up resources held by the common script +common_cleanup diff --git a/ceph-osd/templates/bin/osd/ceph-volume/_common.sh.tpl b/ceph-osd/templates/bin/osd/ceph-volume/_common.sh.tpl new file mode 100644 index 0000000000..fee43d44b4 --- /dev/null +++ b/ceph-osd/templates/bin/osd/ceph-volume/_common.sh.tpl @@ -0,0 +1,562 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +shopt -s expand_aliases +export lock_fd='' +export ALREADY_LOCKED=0 +export PS4='+${BASH_SOURCE:+$(basename ${BASH_SOURCE}):${LINENO}:}${FUNCNAME:+${FUNCNAME}():} ' + +source /tmp/utils-resolveLocations.sh + +: "${CRUSH_LOCATION:=root=default host=${HOSTNAME}}" +: "${OSD_PATH_BASE:=/var/lib/ceph/osd/${CLUSTER}}" +: "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}" +: "${OSD_BOOTSTRAP_KEYRING:=/var/lib/ceph/bootstrap-osd/${CLUSTER}.keyring}" +: "${OSD_JOURNAL_UUID:=$(uuidgen)}" +: "${OSD_JOURNAL_SIZE:=$(awk '/^osd_journal_size/{print $3}' ${CEPH_CONF}.template)}" +: "${OSD_WEIGHT:=1.0}" + +{{ include "helm-toolkit.snippets.mon_host_from_k8s_ep" . }} + +# Obtain a global lock on /var/lib/ceph/tmp/init-osd.lock +function lock() { + # Open a file descriptor for the lock file if there isn't one already + if [[ -z "${lock_fd}" ]]; then + exec {lock_fd}>/var/lib/ceph/tmp/init-osd.lock || exit 1 + fi + flock -w 600 "${lock_fd}" &> /dev/null + ALREADY_LOCKED=1 +} + +# Release the global lock on /var/lib/ceph/tmp/init-osd.lock +function unlock() { + flock -u "${lock_fd}" &> /dev/null + ALREADY_LOCKED=0 +} + +# "Destructor" for common.sh, must be called by scripts that source this one +function common_cleanup() { + # Close the file descriptor for the lock file + if [[ ! -z "${lock_fd}" ]]; then + if [[ ${ALREADY_LOCKED} -ne 0 ]]; then + unlock + fi + eval "exec ${lock_fd}>&-" + fi +} + +# Run a command within the global synchronization lock +function locked() { + # Don't log every command inside locked() to keep logs cleaner + { set +x; } 2>/dev/null + + local LOCK_SCOPE=0 + + # Allow locks to be re-entrant to avoid deadlocks + if [[ ${ALREADY_LOCKED} -eq 0 ]]; then + lock + LOCK_SCOPE=1 + fi + + # Execute the synchronized command + set -x + "$@" + { set +x; } 2>/dev/null + + # Only unlock if the lock was obtained in this scope + if [[ ${LOCK_SCOPE} -ne 0 ]]; then + unlock + fi + + # Re-enable command logging + set -x +} + +# Alias commands that interact with disks so they are always synchronized +alias dmsetup='locked dmsetup' +alias pvs='locked pvs' +alias vgs='locked vgs' +alias lvs='locked lvs' +alias pvdisplay='locked pvdisplay' +alias vgdisplay='locked vgdisplay' +alias lvdisplay='locked lvdisplay' +alias pvcreate='locked pvcreate' +alias vgcreate='locked vgcreate' +alias lvcreate='locked lvcreate' +alias pvremove='locked pvremove' +alias vgremove='locked vgremove' +alias lvremove='locked lvremove' +alias pvrename='locked pvrename' +alias vgrename='locked vgrename' +alias lvrename='locked lvrename' +alias pvchange='locked pvchange' +alias vgchange='locked vgchange' +alias lvchange='locked lvchange' +alias pvscan='locked pvscan' +alias vgscan='locked vgscan' +alias lvscan='locked lvscan' +alias lvm_scan='locked lvm_scan' +alias partprobe='locked partprobe' +alias ceph-volume='locked ceph-volume' +alias disk_zap='locked disk_zap' +alias zap_extra_partitions='locked zap_extra_partitions' +alias udev_settle='locked udev_settle' +alias wipefs='locked wipefs' +alias sgdisk='locked sgdisk' +alias dd='locked dd' + +eval CRUSH_FAILURE_DOMAIN_TYPE=$(cat /etc/ceph/storage.json | python3 -c 'import sys, json; data = json.load(sys.stdin); print(json.dumps(data["failure_domain"]))') +eval CRUSH_FAILURE_DOMAIN_NAME=$(cat /etc/ceph/storage.json | python3 -c 'import sys, json; data = json.load(sys.stdin); print(json.dumps(data["failure_domain_name"]))') +eval CRUSH_FAILURE_DOMAIN_NAME=$(cat /etc/ceph/storage.json | python3 -c 'import sys, json; data = json.load(sys.stdin); print(json.dumps(data["failure_domain_name"]))') +eval CRUSH_FAILURE_DOMAIN_BY_HOSTNAME=$(cat /etc/ceph/storage.json | python3 -c 'import sys, json; data = json.load(sys.stdin); print(json.dumps(data["failure_domain_by_hostname"]))') +eval CRUSH_FAILURE_DOMAIN_FROM_HOSTNAME_MAP=$(cat /etc/ceph/storage.json | jq '.failure_domain_by_hostname_map."'$HOSTNAME'"') +eval DEVICE_CLASS=$(cat /etc/ceph/storage.json | python3 -c 'import sys, json; data = json.load(sys.stdin); print(json.dumps(data["device_class"]))') + +if [[ $(ceph -v | awk '/version/{print $3}' | cut -d. -f1) -lt 12 ]]; then + echo "ERROR - The minimum Ceph version supported is Luminous 12.x.x" + exit 1 +fi + +if [ -z "${HOSTNAME}" ]; then + echo "HOSTNAME not set; This will prevent to add an OSD into the CRUSH map" + exit 1 +fi + +if [[ ! -e ${CEPH_CONF}.template ]]; then + echo "ERROR- ${CEPH_CONF}.template must exist; get it from your existing mon" + exit 1 +else + ENDPOINT=$(mon_host_from_k8s_ep "${NAMESPACE}" ceph-mon-discovery) + if [[ -z "${ENDPOINT}" ]]; then + /bin/sh -c -e "cat ${CEPH_CONF}.template | tee ${CEPH_CONF}" || true + else + /bin/sh -c -e "cat ${CEPH_CONF}.template | sed 's#mon_host.*#mon_host = ${ENDPOINT}#g' | tee ${CEPH_CONF}" || true + fi +fi + +# Wait for a file to exist, regardless of the type +function wait_for_file { + timeout 10 bash -c "while [ ! -e ${1} ]; do echo 'Waiting for ${1} to show up' && sleep 1 ; done" +} + +function is_available { + command -v $@ &>/dev/null +} + +function ceph_cmd_retry() { + cnt=0 + until "ceph" "$@" || [ $cnt -ge 6 ]; do + sleep 10 + ((cnt++)) + done +} + +function crush_create_or_move { + local crush_location=${1} + ceph_cmd_retry --cluster "${CLUSTER}" --name="osd.${OSD_ID}" --keyring="${OSD_KEYRING}" \ + osd crush create-or-move -- "${OSD_ID}" "${OSD_WEIGHT}" ${crush_location} +} + +function crush_add_and_move { + local crush_failure_domain_type=${1} + local crush_failure_domain_name=${2} + local crush_location=$(echo "root=default ${crush_failure_domain_type}=${crush_failure_domain_name} host=${HOSTNAME}") + crush_create_or_move "${crush_location}" + local crush_failure_domain_location_check=$(ceph_cmd_retry --cluster "${CLUSTER}" --name="osd.${OSD_ID}" --keyring="${OSD_KEYRING}" osd find ${OSD_ID} | grep "${crush_failure_domain_type}" | awk -F '"' '{print $4}') + if [ "x${crush_failure_domain_location_check}" != "x${crush_failure_domain_name}" ]; then + # NOTE(supamatt): Manually move the buckets for previously configured CRUSH configurations + # as create-or-move may not appropiately move them. + ceph_cmd_retry --cluster "${CLUSTER}" --name="osd.${OSD_ID}" --keyring="${OSD_KEYRING}" \ + osd crush add-bucket "${crush_failure_domain_name}" "${crush_failure_domain_type}" || true + ceph_cmd_retry --cluster "${CLUSTER}" --name="osd.${OSD_ID}" --keyring="${OSD_KEYRING}" \ + osd crush move "${crush_failure_domain_name}" root=default || true + ceph_cmd_retry --cluster "${CLUSTER}" --name="osd.${OSD_ID}" --keyring="${OSD_KEYRING}" \ + osd crush move "${HOSTNAME}" "${crush_failure_domain_type}=${crush_failure_domain_name}" || true + fi +} + +function crush_location { + set_device_class + if [ "x${CRUSH_FAILURE_DOMAIN_TYPE}" != "xhost" ]; then + + echo "Lets check this host is registered in k8s" + if kubectl get node ${HOSTNAME}; then + CRUSH_FAILURE_DOMAIN_NAME_FROM_NODE_LABEL=$(kubectl get node ${HOSTNAME} -o json| jq -r '.metadata.labels.rack') + else + echo "It seems there is some issue with setting the hostname on this node hence we didnt found this node in k8s" + kubectl get nodes + echo ${HOSTNAME} + exit 1 + fi + + if [ ${CRUSH_FAILURE_DOMAIN_NAME_FROM_NODE_LABEL} != "null" ]; then + CRUSH_FAILURE_DOMAIN_NAME=${CRUSH_FAILURE_DOMAIN_NAME_FROM_NODE_LABEL} + fi + + if [ "x${CRUSH_FAILURE_DOMAIN_NAME}" != "xfalse" ]; then + crush_add_and_move "${CRUSH_FAILURE_DOMAIN_TYPE}" "${CRUSH_FAILURE_DOMAIN_NAME}" + elif [ "x${CRUSH_FAILURE_DOMAIN_BY_HOSTNAME}" != "xfalse" ]; then + crush_add_and_move "${CRUSH_FAILURE_DOMAIN_TYPE}" "$(echo ${CRUSH_FAILURE_DOMAIN_TYPE}_$(echo ${HOSTNAME} | cut -c ${CRUSH_FAILURE_DOMAIN_BY_HOSTNAME}))" + elif [ "x${CRUSH_FAILURE_DOMAIN_FROM_HOSTNAME_MAP}" != "xnull" ]; then + crush_add_and_move "${CRUSH_FAILURE_DOMAIN_TYPE}" "${CRUSH_FAILURE_DOMAIN_FROM_HOSTNAME_MAP}" + else + # NOTE(supamatt): neither variables are defined then we fall back to default behavior + crush_create_or_move "${CRUSH_LOCATION}" + fi + else + crush_create_or_move "${CRUSH_LOCATION}" + fi +} + +# Calculate proper device names, given a device and partition number +function dev_part { + local osd_device=${1} + local osd_partition=${2} + + if [[ -L ${osd_device} ]]; then + # This device is a symlink. Work out it's actual device + local actual_device=$(readlink -f "${osd_device}") + local bn=$(basename "${osd_device}") + if [[ "${actual_device:0-1:1}" == [0-9] ]]; then + local desired_partition="${actual_device}p${osd_partition}" + else + local desired_partition="${actual_device}${osd_partition}" + fi + # Now search for a symlink in the directory of $osd_device + # that has the correct desired partition, and the longest + # shared prefix with the original symlink + local symdir=$(dirname "${osd_device}") + local link="" + local pfxlen=0 + for option in ${symdir}/*; do + [[ -e $option ]] || break + if [[ $(readlink -f "${option}") == "${desired_partition}" ]]; then + local optprefixlen=$(prefix_length "${option}" "${bn}") + if [[ ${optprefixlen} > ${pfxlen} ]]; then + link=${symdir}/${option} + pfxlen=${optprefixlen} + fi + fi + done + if [[ $pfxlen -eq 0 ]]; then + >&2 echo "Could not locate appropriate symlink for partition ${osd_partition} of ${osd_device}" + exit 1 + fi + echo "$link" + elif [[ "${osd_device:0-1:1}" == [0-9] ]]; then + echo "${osd_device}p${osd_partition}" + else + echo "${osd_device}${osd_partition}" + fi +} + +function zap_extra_partitions { + # Examine temp mount and delete any block.db and block.wal partitions + mountpoint=${1} + journal_disk="" + journal_part="" + block_db_disk="" + block_db_part="" + block_wal_disk="" + block_wal_part="" + + # Discover journal, block.db, and block.wal partitions first before deleting anything + # If the partitions are on the same disk, deleting one can affect discovery of the other(s) + if [ -L "${mountpoint}/journal" ]; then + journal_disk=$(readlink -m ${mountpoint}/journal | sed 's/[0-9]*//g') + journal_part=$(readlink -m ${mountpoint}/journal | sed 's/[^0-9]*//g') + fi + if [ -L "${mountpoint}/block.db" ]; then + block_db_disk=$(readlink -m ${mountpoint}/block.db | sed 's/[0-9]*//g') + block_db_part=$(readlink -m ${mountpoint}/block.db | sed 's/[^0-9]*//g') + fi + if [ -L "${mountpoint}/block.wal" ]; then + block_wal_disk=$(readlink -m ${mountpoint}/block.wal | sed 's/[0-9]*//g') + block_wal_part=$(readlink -m ${mountpoint}/block.wal | sed 's/[^0-9]*//g') + fi + + # Delete any discovered journal, block.db, and block.wal partitions + if [ ! -z "${journal_disk}" ]; then + sgdisk -d ${journal_part} ${journal_disk} + /usr/bin/flock -s ${journal_disk} /sbin/partprobe ${journal_disk} + fi + if [ ! -z "${block_db_disk}" ]; then + sgdisk -d ${block_db_part} ${block_db_disk} + /usr/bin/flock -s ${block_db_disk} /sbin/partprobe ${block_db_disk} + fi + if [ ! -z "${block_wal_disk}" ]; then + sgdisk -d ${block_wal_part} ${block_wal_disk} + /usr/bin/flock -s ${block_wal_disk} /sbin/partprobe ${block_wal_disk} + fi +} + +function disk_zap { + # Run all the commands to clear a disk + local device=${1} + local dm_devices=$(get_dm_devices_from_osd_device "${device}" | xargs) + for dm_device in ${dm_devices}; do + if [[ "$(dmsetup ls | grep ${dm_device})" ]]; then + dmsetup remove ${dm_device} + fi + done + local logical_volumes=$(get_lv_paths_from_osd_device "${device}" | xargs) + if [[ "${logical_volumes}" ]]; then + lvremove -y ${logical_volumes} + fi + local volume_group=$(pvdisplay -ddd -v ${device} | grep "VG Name" | awk '/ceph/{print $3}' | grep "ceph") + if [[ ${volume_group} ]]; then + vgremove -y ${volume_group} + pvremove -y ${device} + ceph-volume lvm zap ${device} --destroy + fi + wipefs --all ${device} + sgdisk --zap-all -- ${device} + # Wipe the first 200MB boundary, as Bluestore redeployments will not work otherwise + dd if=/dev/zero of=${device} bs=1M count=200 +} + +# This should be run atomically to prevent unexpected cache states +function lvm_scan { + pvscan --cache + vgscan --cache + lvscan --cache + pvscan + vgscan + lvscan +} + +function wait_for_device { + local device="$1" + + echo "Waiting for block device ${device} to appear" + for countdown in {1..600}; do + test -b "${device}" && break + sleep 1 + done + test -b "${device}" || exit 1 +} + +function udev_settle { + osd_devices="${OSD_DEVICE}" + partprobe "${OSD_DEVICE}" + lvm_scan + if [ "${OSD_BLUESTORE:-0}" -eq 1 ]; then + if [ ! -z "$BLOCK_DB" ]; then + osd_devices="${osd_devices}\|${BLOCK_DB}" + # BLOCK_DB could be a physical or logical device here + local block_db="$BLOCK_DB" + local db_vg="$(echo $block_db | cut -d'/' -f1)" + if [ ! -z "$db_vg" ]; then + block_db=$(pvdisplay -ddd -v | grep -B1 "$db_vg" | awk '/PV Name/{print $3}') + fi + partprobe "${block_db}" + fi + if [ ! -z "$BLOCK_WAL" ] && [ "$BLOCK_WAL" != "$BLOCK_DB" ]; then + osd_devices="${osd_devices}\|${BLOCK_WAL}" + # BLOCK_WAL could be a physical or logical device here + local block_wal="$BLOCK_WAL" + local wal_vg="$(echo $block_wal | cut -d'/' -f1)" + if [ ! -z "$wal_vg" ]; then + block_wal=$(pvdisplay -ddd -v | grep -B1 "$wal_vg" | awk '/PV Name/{print $3}') + fi + partprobe "${block_wal}" + fi + else + if [ "x$JOURNAL_TYPE" == "xblock-logical" ] && [ ! -z "$OSD_JOURNAL" ]; then + OSD_JOURNAL=$(readlink -f ${OSD_JOURNAL}) + if [ ! -z "$OSD_JOURNAL" ]; then + local JDEV=$(echo ${OSD_JOURNAL} | sed 's/[0-9]//g') + osd_devices="${osd_devices}\|${JDEV}" + partprobe "${JDEV}" + wait_for_device "${JDEV}" + fi + fi + fi + + # On occassion udev may not make the correct device symlinks for Ceph, just in case we make them manually + mkdir -p /dev/disk/by-partuuid + for dev in $(awk '!/rbd/{print $4}' /proc/partitions | grep "${osd_devices}" | grep "[0-9]"); do + diskdev=$(echo "${dev//[!a-z]/}") + partnum=$(echo "${dev//[!0-9]/}") + symlink="/dev/disk/by-partuuid/$(sgdisk -i ${partnum} /dev/${diskdev} | awk '/Partition unique GUID/{print tolower($4)}')" + if [ ! -e "${symlink}" ]; then + ln -s "../../${dev}" "${symlink}" + fi + done +} + +# Helper function to get a logical volume from a physical volume +function get_lv_from_device { + device="$1" + + pvdisplay -ddd -v -m ${device} | awk '/Logical volume/{print $3}' +} + +# Helper function to get an lvm tag from a logical volume +function get_lvm_tag_from_volume { + logical_volume="$1" + tag="$2" + + if [[ "$#" -lt 2 ]] || [[ -z "${logical_volume}" ]]; then + # Return an empty string if the logical volume doesn't exist + echo + else + # Get and return the specified tag from the logical volume + lvs -o lv_tags ${logical_volume} | tr ',' '\n' | grep ${tag} | cut -d'=' -f2 + fi +} + +# Helper function to get an lvm tag from a physical device +function get_lvm_tag_from_device { + device="$1" + tag="$2" + # Attempt to get a logical volume for the physical device + logical_volume="$(get_lv_from_device ${device})" + + # Use get_lvm_tag_from_volume to get the specified tag from the logical volume + get_lvm_tag_from_volume ${logical_volume} ${tag} +} + +# Helper function to get the size of a logical volume +function get_lv_size_from_device { + device="$1" + logical_volume="$(get_lv_from_device ${device})" + + lvs ${logical_volume} -o LV_SIZE --noheadings --units k --nosuffix | xargs | cut -d'.' -f1 +} + +# Helper function to get the crush weight for an osd device +function get_osd_crush_weight_from_device { + device="$1" + lv_size="$(get_lv_size_from_device ${device})" # KiB + + if [[ ! -z "${BLOCK_DB_SIZE}" ]]; then + db_size=$(echo "${BLOCK_DB_SIZE}" | cut -d'B' -f1 | numfmt --from=iec | awk '{print $1/1024}') # KiB + lv_size=$((lv_size+db_size)) # KiB + fi + + echo ${lv_size} | awk '{printf("%.2f\n", $1/1073741824)}' # KiB to TiB +} + +# Helper function to get a cluster FSID from a physical device +function get_cluster_fsid_from_device { + device="$1" + + # Use get_lvm_tag_from_device to get the cluster FSID from the device + get_lvm_tag_from_device ${device} ceph.cluster_fsid +} + +# Helper function to get an OSD ID from a logical volume +function get_osd_id_from_volume { + logical_volume="$1" + + # Use get_lvm_tag_from_volume to get the OSD ID from the logical volume + get_lvm_tag_from_volume ${logical_volume} ceph.osd_id +} + +# Helper function get an OSD ID from a physical device +function get_osd_id_from_device { + device="$1" + + # Use get_lvm_tag_from_device to get the OSD ID from the device + get_lvm_tag_from_device ${device} ceph.osd_id +} + +# Helper function get an OSD FSID from a physical device +function get_osd_fsid_from_device { + device="$1" + + # Use get_lvm_tag_from_device to get the OSD FSID from the device + get_lvm_tag_from_device ${device} ceph.osd_fsid +} + +# Helper function get an OSD DB device from a physical device +function get_osd_db_device_from_device { + device="$1" + + # Use get_lvm_tag_from_device to get the OSD DB device from the device + get_lvm_tag_from_device ${device} ceph.db_device +} + +# Helper function get an OSD WAL device from a physical device +function get_osd_wal_device_from_device { + device="$1" + + # Use get_lvm_tag_from_device to get the OSD WAL device from the device + get_lvm_tag_from_device ${device} ceph.wal_device +} + +function get_block_uuid_from_device { + device="$1" + + get_lvm_tag_from_device ${device} ceph.block_uuid +} + +function get_dm_devices_from_osd_device { + device="$1" + pv_uuid=$(pvdisplay -ddd -v ${device} | awk '/PV UUID/{print $3}') + + # Return the list of dm devices that belong to the osd + if [[ "${pv_uuid}" ]]; then + dmsetup ls | grep "$(echo "${pv_uuid}" | sed 's/-/--/g')" | awk '{print $1}' + fi +} + +function get_lv_paths_from_osd_device { + device="$1" + pv_uuid=$(pvdisplay -ddd -v ${device} | awk '/PV UUID/{print $3}') + + # Return the list of lvs that belong to the osd + if [[ "${pv_uuid}" ]]; then + lvdisplay | grep "LV Path" | grep "${pv_uuid}" | awk '{print $3}' + fi +} + +function get_vg_name_from_device { + device="$1" + pv_uuid=$(pvdisplay -ddd -v ${device} | awk '/PV UUID/{print $3}') + + if [[ "${pv_uuid}" ]]; then + echo "ceph-vg-${pv_uuid}" + fi +} + +function get_lv_name_from_device { + device="$1" + device_type="$2" + pv_uuid=$(pvdisplay -ddd -v ${device} | awk '/PV UUID/{print $3}') + + if [[ "${pv_uuid}" ]]; then + echo "ceph-${device_type}-${pv_uuid}" + fi +} + +function set_device_class { + if [ ! -z "$DEVICE_CLASS" ]; then + if [ "x$DEVICE_CLASS" != "x$(get_device_class)" ]; then + ceph_cmd_retry --cluster "${CLUSTER}" --name="osd.${OSD_ID}" --keyring="${OSD_KEYRING}" \ + osd crush rm-device-class "osd.${OSD_ID}" + ceph_cmd_retry --cluster "${CLUSTER}" --name="osd.${OSD_ID}" --keyring="${OSD_KEYRING}" \ + osd crush set-device-class "${DEVICE_CLASS}" "osd.${OSD_ID}" + fi + fi +} + +function get_device_class { + echo $(ceph_cmd_retry --cluster "${CLUSTER}" --name="osd.${OSD_ID}" --keyring="${OSD_KEYRING}" \ + osd crush get-device-class "osd.${OSD_ID}") +} diff --git a/ceph-osd/templates/bin/osd/ceph-volume/_init-ceph-volume-helper-block-logical.sh.tpl b/ceph-osd/templates/bin/osd/ceph-volume/_init-ceph-volume-helper-block-logical.sh.tpl new file mode 100644 index 0000000000..3154a73c6d --- /dev/null +++ b/ceph-osd/templates/bin/osd/ceph-volume/_init-ceph-volume-helper-block-logical.sh.tpl @@ -0,0 +1,214 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +# We do not want to zap journal disk. Tracking this option seperatly. +: "${JOURNAL_FORCE_ZAP:=0}" + +export OSD_DEVICE=$(readlink -f ${STORAGE_LOCATION}) +export OSD_BLUESTORE=0 + +if [ "x$JOURNAL_TYPE" == "xdirectory" ]; then + export OSD_JOURNAL="/var/lib/ceph/journal" +else + export OSD_JOURNAL=$(readlink -f ${JOURNAL_LOCATION}) +fi + +# Check OSD FSID and journalling metadata +# Returns 1 if the disk should be zapped; 0 otherwise. +function check_osd_metadata { + local ceph_fsid=$1 + retcode=0 + local tmpmnt=$(mktemp -d) + mount ${DM_DEV} ${tmpmnt} + + if [ "x${JOURNAL_TYPE}" != "xdirectory" ]; then + if [ -f "${tmpmnt}/whoami" ]; then + OSD_JOURNAL_DISK=$(readlink -f "${tmpmnt}/journal") + local osd_id=$(cat "${tmpmnt}/whoami") + if [ ! -b "${OSD_JOURNAL_DISK}" ]; then + OSD_JOURNAL=$(readlink -f ${OSD_JOURNAL}) + local jdev=$(echo ${OSD_JOURNAL} | sed 's/[0-9]//g') + if [ ${jdev} == ${OSD_JOURNAL} ]; then + echo "OSD Init: It appears that ${OSD_DEVICE} is missing the journal at ${OSD_JOURNAL}." + echo "OSD Init: Because OSD_FORCE_REPAIR is set, we will wipe the metadata of the OSD and zap it." + rm -rf ${tmpmnt}/ceph_fsid + else + echo "OSD Init: It appears that ${OSD_DEVICE} is missing the journal at ${OSD_JOURNAL_DISK}." + echo "OSD Init: Because OSD_FORCE_REPAIR is set and paritions are manually defined, we will" + echo "OSD Init: attempt to recreate the missing journal device partitions." + osd_journal_create ${OSD_JOURNAL} + ln -sf /dev/disk/by-partuuid/${OSD_JOURNAL_UUID} ${tmpmnt}/journal + echo ${OSD_JOURNAL_UUID} | tee ${tmpmnt}/journal_uuid + chown ceph. ${OSD_JOURNAL} + # During OSD start we will format the journal and set the fsid + touch ${tmpmnt}/run_mkjournal + fi + fi + else + echo "OSD Init: It looks like ${OSD_DEVICE} has a ceph data partition but is missing it's metadata." + echo "OSD Init: The device may contain inconsistent metadata or be corrupted." + echo "OSD Init: Because OSD_FORCE_REPAIR is set, we will wipe the metadata of the OSD and zap it." + rm -rf ${tmpmnt}/ceph_fsid + fi + fi + + if [ -f "${tmpmnt}/ceph_fsid" ]; then + local osd_fsid=$(cat "${tmpmnt}/ceph_fsid") + + if [ ${osd_fsid} != ${ceph_fsid} ]; then + echo "OSD Init: ${OSD_DEVICE} is an OSD belonging to a different (or old) ceph cluster." + echo "OSD Init: The OSD FSID is ${osd_fsid} while this cluster is ${ceph_fsid}" + echo "OSD Init: Because OSD_FORCE_REPAIR was set, we will zap this device." + ZAP_EXTRA_PARTITIONS=${tmpmnt} + retcode=1 + else + echo "It looks like ${OSD_DEVICE} is an OSD belonging to a this ceph cluster." + echo "OSD_FORCE_REPAIR is set, but will be ignored and the device will not be zapped." + echo "Moving on, trying to activate the OSD now." + fi + else + echo "OSD Init: ${OSD_DEVICE} has a ceph data partition but no FSID." + echo "OSD Init: Because OSD_FORCE_REPAIR was set, we will zap this device." + ZAP_EXTRA_PARTITIONS=${tmpmnt} + retcode=1 + fi + umount ${tmpmnt} + return ${retcode} +} + +function determine_what_needs_zapping { + + if [[ ! -z ${OSD_ID} ]]; then + local dm_num=$(dmsetup ls | grep $(lsblk -J ${OSD_DEVICE} | jq -r '.blockdevices[].children[].name') | awk '{print $2}' | cut -d':' -f2 | cut -d')' -f1) + DM_DEV="/dev/dm-"${dm_num} + elif [[ $(sgdisk --print ${OSD_DEVICE} | grep "F800") ]]; then + # Ceph-disk was used to initialize the disk, but this is not supported + echo "OSD Init: ceph-disk was used to initialize the disk, but this is no longer supported" + exit 1 + else + if [[ ${OSD_FORCE_REPAIR} -eq 1 ]]; then + echo "OSD Init: It looks like ${OSD_DEVICE} isn't consistent, however OSD_FORCE_REPAIR is enabled so we are zapping the device anyway" + ZAP_DEVICE=1 + else + echo "OSD Init: Regarding parted, device ${OSD_DEVICE} is inconsistent/broken/weird." + echo "OSD Init: It would be too dangerous to destroy it without any notification." + echo "OSD Init: Please set OSD_FORCE_REPAIR to '1' if you really want to zap this disk." + exit 1 + fi + fi + + if [ ${OSD_FORCE_REPAIR} -eq 1 ] && [ ! -z ${DM_DEV} ]; then + if [ -b ${DM_DEV} ]; then + local ceph_fsid=$(ceph-conf --lookup fsid) + if [ ! -z "${ceph_fsid}" ]; then + # Check the OSD metadata and zap the disk if necessary + if [[ $(check_osd_metadata ${ceph_fsid}) -eq 1 ]]; then + echo "OSD Init: ${OSD_DEVICE} needs to be zapped..." + ZAP_DEVICE=1 + fi + else + echo "Unable to determine the FSID of the current cluster." + echo "OSD_FORCE_REPAIR is set, but this OSD will not be zapped." + echo "Moving on, trying to activate the OSD now." + fi + else + echo "parted says ${DM_DEV} should exist, but we do not see it." + echo "We will ignore OSD_FORCE_REPAIR and try to use the device as-is" + echo "Moving on, trying to activate the OSD now." + fi + else + echo "INFO- It looks like ${OSD_DEVICE} is an OSD LVM" + echo "Moving on, trying to prepare and activate the OSD LVM now." + fi +} + +function osd_journal_create { + local osd_journal=${1} + local osd_journal_partition=$(echo ${osd_journal} | sed 's/[^0-9]//g') + local jdev=$(echo ${osd_journal} | sed 's/[0-9]//g') + if [ -b "${jdev}" ]; then + sgdisk --new=${osd_journal_partition}:0:+${OSD_JOURNAL_SIZE}M \ + --change-name='${osd_journal_partition}:ceph journal' \ + --partition-guid=${osd_journal_partition}:${OSD_JOURNAL_UUID} \ + --typecode=${osd_journal_partition}:45b0969e-9b03-4f30-b4c6-b4b80ceff106 --mbrtogpt -- ${jdev} + OSD_JOURNAL=$(dev_part ${jdev} ${osd_journal_partition}) + udev_settle + else + echo "OSD Init: The backing device ${jdev} for ${OSD_JOURNAL} does not exist on this system." + exit 1 + fi +} + +function osd_journal_prepare { + if [ -n "${OSD_JOURNAL}" ]; then + if [ -b ${OSD_JOURNAL} ]; then + OSD_JOURNAL=$(readlink -f ${OSD_JOURNAL}) + OSD_JOURNAL_PARTITION=$(echo ${OSD_JOURNAL} | sed 's/[^0-9]//g') + local jdev=$(echo ${OSD_JOURNAL} | sed 's/[0-9]//g') + if [ -z "${OSD_JOURNAL_PARTITION}" ]; then + OSD_JOURNAL=$(dev_part ${jdev} ${OSD_JOURNAL_PARTITION}) + else + OSD_JOURNAL=${OSD_JOURNAL} + fi + elif [ "x${JOURNAL_TYPE}" != "xdirectory" ]; then + # The block device exists but doesn't appear to be paritioned, we will proceed with parititioning the device. + OSD_JOURNAL=$(readlink -f ${OSD_JOURNAL}) + until [ -b ${OSD_JOURNAL} ]; do + osd_journal_create ${OSD_JOURNAL} + done + fi + chown ceph. ${OSD_JOURNAL}; + elif [ "x${JOURNAL_TYPE}" != "xdirectory" ]; then + echo "No journal device specified. OSD and journal will share ${OSD_DEVICE}" + echo "For better performance on HDD, consider moving your journal to a separate device" + fi + CLI_OPTS="${CLI_OPTS} --filestore" +} + +function osd_disk_prepare { + + if [[ ${CEPH_LVM_PREPARE} -eq 1 ]] || [[ ${DISK_ZAPPED} -eq 1 ]]; then + udev_settle + RESULTING_VG=""; RESULTING_LV=""; + create_vg_if_needed "${OSD_DEVICE}" + create_lv_if_needed "${OSD_DEVICE}" "${RESULTING_VG}" "--yes -l 100%FREE" + + CLI_OPTS="${CLI_OPTS} --data ${RESULTING_LV}" + CEPH_LVM_PREPARE=1 + udev_settle + fi + if pvdisplay -ddd -v ${OSD_DEVICE} | awk '/VG Name/{print $3}' | grep "ceph"; then + echo "OSD Init: Device is already set up. LVM prepare does not need to be called." + CEPH_LVM_PREPARE=0 + fi + + osd_journal_prepare + CLI_OPTS="${CLI_OPTS} --data ${OSD_DEVICE} --journal ${OSD_JOURNAL}" + udev_settle + + if [ ! -z "$DEVICE_CLASS" ]; then + CLI_OPTS="${CLI_OPTS} --crush-device-class ${DEVICE_CLASS}" + fi + + if [[ ${CEPH_LVM_PREPARE} -eq 1 ]]; then + echo "OSD Init: Calling ceph-volume lvm-v prepare ${CLI_OPTS}" + ceph-volume lvm -v prepare ${CLI_OPTS} + udev_settle + fi +} + diff --git a/ceph-osd/templates/bin/osd/ceph-volume/_init-ceph-volume-helper-bluestore.sh.tpl b/ceph-osd/templates/bin/osd/ceph-volume/_init-ceph-volume-helper-bluestore.sh.tpl new file mode 100644 index 0000000000..44f22284d9 --- /dev/null +++ b/ceph-osd/templates/bin/osd/ceph-volume/_init-ceph-volume-helper-bluestore.sh.tpl @@ -0,0 +1,176 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +export OSD_DEVICE=$(readlink -f ${STORAGE_LOCATION}) +export OSD_BLUESTORE=1 +alias prep_device='locked prep_device' + +function check_block_device_for_zap { + local block_device=$1 + local device_type=$2 + + if [[ ${block_device} ]]; then + local vg_name=$(get_vg_name_from_device ${block_device}) + local lv_name=$(get_lv_name_from_device ${OSD_DEVICE} ${device_type}) + local vg=$(vgs --noheadings -o vg_name -S "vg_name=${vg_name}" | tr -d '[:space:]') + if [[ "${vg}" ]]; then + local device_osd_id=$(get_osd_id_from_volume "/dev/${vg_name}/${lv_name}") + CEPH_LVM_PREPARE=1 + if [[ -n "${device_osd_id}" ]] && [[ -n "${OSD_ID}" ]]; then + if [[ "${device_osd_id}" == "${OSD_ID}" ]]; then + echo "OSD Init: OSD ID matches the OSD ID already on the data volume. LVM prepare does not need to be called." + CEPH_LVM_PREPARE=0 + else + echo "OSD Init: OSD ID does match the OSD ID on the data volume. Device needs to be zapped." + ZAP_DEVICE=1 + fi + fi + + # Check if this device (db or wal) has no associated data volume + local logical_volumes="$(lvs --noheadings -o lv_name ${vg} | xargs)" + for volume in ${logical_volumes}; do + local data_volume=$(echo ${volume} | sed -E -e 's/-db-|-wal-/-lv-/g') + if [[ -z $(lvs --noheadings -o lv_name -S "lv_name=${data_volume}") ]]; then + # DB or WAL volume without a corresponding data volume, remove it + lvremove -y /dev/${vg}/${volume} + echo "OSD Init: LV /dev/${vg}/${volume} was removed as it did not have a data volume." + fi + done + else + if [[ "${vg_name}" ]]; then + local logical_devices=$(get_dm_devices_from_osd_device "${OSD_DEVICE}") + local device_filter=$(echo "${vg_name}" | sed 's/-/--/g') + local logical_devices=$(echo "${logical_devices}" | grep "${device_filter}" | xargs) + if [[ "$logical_devices" ]]; then + echo "OSD Init: No VG resources found with name ${vg_name}. Device needs to be zapped." + ZAP_DEVICE=1 + fi + fi + fi + fi +} + +function determine_what_needs_zapping { + + local osd_fsid=$(get_cluster_fsid_from_device ${OSD_DEVICE}) + local cluster_fsid=$(ceph-conf --lookup fsid) + + # If the OSD FSID is defined within the device, check if we're already bootstrapped. + if [[ ! -z "${osd_fsid}" ]]; then + # Check if the OSD FSID is the same as the cluster FSID. If so, then we're + # already bootstrapped; otherwise, this is an old disk and needs to + # be zapped. + if [[ "${osd_fsid}" == "${cluster_fsid}" ]]; then + if [[ ! -z "${OSD_ID}" ]]; then + # Check to see what needs to be done to prepare the disk. If the OSD + # ID is in the Ceph OSD list, then LVM prepare does not need to be done. + if ceph --name client.bootstrap-osd --keyring $OSD_BOOTSTRAP_KEYRING osd ls |grep -w ${OSD_ID}; then + echo "OSD Init: Running bluestore mode and ${OSD_DEVICE} already bootstrapped. LVM prepare does not need to be called." + CEPH_LVM_PREPARE=0 + elif [[ ${OSD_FORCE_REPAIR} -eq 1 ]]; then + echo "OSD initialized for this cluster, but OSD ID not found in the cluster, reinitializing" + ZAP_DEVICE=1 + else + echo "OSD initialized for this cluster, but OSD ID not found in the cluster, repair manually" + fi + fi + else + echo "OSD Init: OSD FSID ${osd_fsid} initialized for a different cluster. It needs to be zapped." + ZAP_DEVICE=1 + fi + elif [[ $(sgdisk --print ${OSD_DEVICE} | grep "F800") ]]; then + # Ceph-disk was used to initialize the disk, but this is not supported + echo "ceph-disk was used to initialize the disk, but this is no longer supported" + exit 1 + fi + + check_block_device_for_zap "${BLOCK_DB}" db + check_block_device_for_zap "${BLOCK_WAL}" wal + + # Zapping extra partitions isn't done for bluestore + ZAP_EXTRA_PARTITIONS=0 +} + +function prep_device { + local block_device=$1 + local block_device_size=$2 + local device_type=$3 + local vg_name lv_name vg device_osd_id logical_devices logical_volume + RESULTING_VG=""; RESULTING_LV=""; + + udev_settle + vg_name=$(get_vg_name_from_device ${block_device}) + lv_name=$(get_lv_name_from_device ${OSD_DEVICE} ${device_type}) + vg=$(vgs --noheadings -o vg_name -S "vg_name=${vg_name}" | tr -d '[:space:]') + if [[ -z "${vg}" ]]; then + create_vg_if_needed "${block_device}" + vg=${RESULTING_VG} + fi + udev_settle + + create_lv_if_needed "${block_device}" "${vg}" "--yes -L ${block_device_size}" "${lv_name}" + if [[ "${device_type}" == "db" ]]; then + BLOCK_DB=${RESULTING_LV} + elif [[ "${device_type}" == "wal" ]]; then + BLOCK_WAL=${RESULTING_LV} + fi + udev_settle +} + +function osd_disk_prepare { + + if [[ ${CEPH_LVM_PREPARE} -eq 1 ]] || [[ ${DISK_ZAPPED} -eq 1 ]]; then + udev_settle + RESULTING_VG=""; RESULTING_LV=""; + create_vg_if_needed "${OSD_DEVICE}" + create_lv_if_needed "${OSD_DEVICE}" "${RESULTING_VG}" "--yes -l 100%FREE" + + CLI_OPTS="${CLI_OPTS} --data ${RESULTING_LV}" + CEPH_LVM_PREPARE=1 + udev_settle + fi + + if [[ ${BLOCK_DB} && ${BLOCK_WAL} ]]; then + prep_device "${BLOCK_DB}" "${BLOCK_DB_SIZE}" "db" "${OSD_DEVICE}" + prep_device "${BLOCK_WAL}" "${BLOCK_WAL_SIZE}" "wal" "${OSD_DEVICE}" + elif [[ -z ${BLOCK_DB} && ${BLOCK_WAL} ]]; then + prep_device "${BLOCK_WAL}" "${BLOCK_WAL_SIZE}" "wal" "${OSD_DEVICE}" + elif [[ ${BLOCK_DB} && -z ${BLOCK_WAL} ]]; then + prep_device "${BLOCK_DB}" "${BLOCK_DB_SIZE}" "db" "${OSD_DEVICE}" + fi + + CLI_OPTS="${CLI_OPTS} --bluestore" + + if [ ! -z "$BLOCK_DB" ]; then + CLI_OPTS="${CLI_OPTS} --block.db ${BLOCK_DB}" + fi + + if [ ! -z "$BLOCK_WAL" ]; then + CLI_OPTS="${CLI_OPTS} --block.wal ${BLOCK_WAL}" + fi + + if [ ! -z "$DEVICE_CLASS" ]; then + CLI_OPTS="${CLI_OPTS} --crush-device-class ${DEVICE_CLASS}" + fi + + if [[ ${CEPH_LVM_PREPARE} -eq 1 ]]; then + echo "OSD Init: Calling ceph-volume lvm-v prepare ${CLI_OPTS}" + ceph-volume lvm -v prepare ${CLI_OPTS} + udev_settle + fi +} diff --git a/ceph-osd/templates/bin/osd/ceph-volume/_init-ceph-volume-helper-directory.sh.tpl b/ceph-osd/templates/bin/osd/ceph-volume/_init-ceph-volume-helper-directory.sh.tpl new file mode 100644 index 0000000000..151766b438 --- /dev/null +++ b/ceph-osd/templates/bin/osd/ceph-volume/_init-ceph-volume-helper-directory.sh.tpl @@ -0,0 +1,23 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +# We do not want to zap journal disk. Tracking this option seperatly. +: "${JOURNAL_FORCE_ZAP:=0}" + +export OSD_DEVICE="/var/lib/ceph/osd" +export OSD_JOURNAL="/var/lib/ceph/journal" diff --git a/ceph-osd/templates/bin/osd/ceph-volume/_init-with-ceph-volume.sh.tpl b/ceph-osd/templates/bin/osd/ceph-volume/_init-with-ceph-volume.sh.tpl new file mode 100644 index 0000000000..77fa74b944 --- /dev/null +++ b/ceph-osd/templates/bin/osd/ceph-volume/_init-with-ceph-volume.sh.tpl @@ -0,0 +1,268 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +: "${OSD_FORCE_REPAIR:=0}" + +source /tmp/osd-common-ceph-volume.sh + +source /tmp/init-ceph-volume-helper-${STORAGE_TYPE}.sh + + +# Set up aliases for functions that require disk synchronization +alias rename_vg='locked rename_vg' +alias rename_lvs='locked rename_lvs' +alias update_lv_tags='locked update_lv_tags' + +# Renames a single VG if necessary +function rename_vg { + local physical_disk=$1 + local old_vg_name=$(pvdisplay -ddd -v ${physical_disk} | awk '/VG Name/{print $3}') + local vg_name=$(get_vg_name_from_device ${physical_disk}) + + if [[ "${old_vg_name}" ]] && [[ "${vg_name}" != "${old_vg_name}" ]]; then + vgrename ${old_vg_name} ${vg_name} + echo "OSD Init: Renamed volume group ${old_vg_name} to ${vg_name}." + fi +} + +# Renames all LVs associated with an OSD as necesasry +function rename_lvs { + local data_disk=$1 + local vg_name=$(pvdisplay -ddd -v ${data_disk} | awk '/VG Name/{print $3}') + + if [[ "${vg_name}" ]]; then + # Rename the OSD volume if necessary + local old_lv_name=$(lvdisplay ${vg_name} | awk '/LV Name/{print $3}') + local lv_name=$(get_lv_name_from_device ${data_disk} lv) + + if [[ "${old_lv_name}" ]] && [[ "${lv_name}" != "${old_lv_name}" ]]; then + lvrename ${vg_name} ${old_lv_name} ${lv_name} + echo "OSD Init: Renamed logical volume ${old_lv_name} (from group ${vg_name}) to ${lv_name}." + fi + + # Rename the OSD's block.db volume if necessary, referenced by UUID + local lv_tag=$(get_lvm_tag_from_device ${data_disk} ceph.db_uuid) + + if [[ "${lv_tag}" ]]; then + local lv_device=$(lvdisplay | grep -B4 "${lv_tag}" | awk '/LV Path/{print $3}') + + if [[ "${lv_device}" ]]; then + local db_vg=$(echo ${lv_device} | awk -F "/" '{print $3}') + old_lv_name=$(echo ${lv_device} | awk -F "/" '{print $4}') + local db_name=$(get_lv_name_from_device ${data_disk} db) + + if [[ "${old_lv_name}" ]] && [[ "${db_name}" != "${old_lv_name}" ]]; then + lvrename ${db_vg} ${old_lv_name} ${db_name} + echo "OSD Init: Renamed DB logical volume ${old_lv_name} (from group ${db_vg}) to ${db_name}." + fi + fi + fi + + # Rename the OSD's WAL volume if necessary, referenced by UUID + lv_tag=$(get_lvm_tag_from_device ${data_disk} ceph.wal_uuid) + + if [[ "${lv_tag}" ]]; then + local lv_device=$(lvdisplay | grep -B4 "${lv_tag}" | awk '/LV Path/{print $3}') + + if [[ "${lv_device}" ]]; then + local wal_vg=$(echo ${lv_device} | awk -F "/" '{print $3}') + old_lv_name=$(echo ${lv_device} | awk -F "/" '{print $4}') + local wal_name=$(get_lv_name_from_device ${data_disk} wal) + + if [[ "${old_lv_name}" ]] && [[ "${wal_name}" != "${old_lv_name}" ]]; then + lvrename ${wal_vg} ${old_lv_name} ${wal_name} + echo "OSD Init: Renamed WAL logical volume ${old_lv_name} (from group ${wal_vg}) to ${wal_name}." + fi + fi + fi + fi +} + +# Fixes up the tags that reference block, db, and wal logical_volumes +# NOTE: This updates tags based on current VG and LV names, so any necessary +# renaming should be completed prior to calling this +function update_lv_tags { + local data_disk=$1 + local pv_uuid=$(pvdisplay -ddd -v ${data_disk} | awk '/PV UUID/{print $3}') + + if [[ "${pv_uuid}" ]]; then + local volumes="$(lvs --no-headings | grep -e "${pv_uuid}")" + local block_device db_device wal_device vg_name + local old_block_device old_db_device old_wal_device + + # Build OSD device paths from current VG and LV names + while read lv vg other_stuff; do + if [[ "${lv}" == "$(get_lv_name_from_device ${data_disk} lv)" ]]; then + block_device="/dev/${vg}/${lv}" + old_block_device=$(get_lvm_tag_from_volume ${block_device} ceph.block_device) + fi + if [[ "${lv}" == "$(get_lv_name_from_device ${data_disk} db)" ]]; then + db_device="/dev/${vg}/${lv}" + old_db_device=$(get_lvm_tag_from_volume ${block_device} ceph.db_device) + fi + if [[ "${lv}" == "$(get_lv_name_from_device ${data_disk} wal)" ]]; then + wal_device="/dev/${vg}/${lv}" + old_wal_device=$(get_lvm_tag_from_volume ${block_device} ceph.wal_device) + fi + done <<< ${volumes} + + # Set new tags on all of the volumes using paths built above + while read lv vg other_stuff; do + if [[ "${block_device}" ]]; then + if [[ "${old_block_device}" ]]; then + lvchange --deltag "ceph.block_device=${old_block_device}" /dev/${vg}/${lv} + fi + lvchange --addtag "ceph.block_device=${block_device}" /dev/${vg}/${lv} + echo "OSD Init: Updated lv tags for data volume ${block_device}." + fi + if [[ "${db_device}" ]]; then + if [[ "${old_db_device}" ]]; then + lvchange --deltag "ceph.db_device=${old_db_device}" /dev/${vg}/${lv} + fi + lvchange --addtag "ceph.db_device=${db_device}" /dev/${vg}/${lv} + echo "OSD Init: Updated lv tags for DB volume ${db_device}." + fi + if [[ "${wal_device}" ]]; then + if [[ "${old_wal_device}" ]]; then + lvchange --deltag "ceph.wal_device=${old_wal_device}" /dev/${vg}/${lv} + fi + lvchange --addtag "ceph.wal_device=${wal_device}" /dev/${vg}/${lv} + echo "OSD Init: Updated lv tags for WAL volume ${wal_device}." + fi + done <<< ${volumes} + fi +} + +function create_vg_if_needed { + local bl_device=$1 + local vg_name=$(get_vg_name_from_device ${bl_device}) + if [[ -z "${vg_name}" ]]; then + local random_uuid=$(uuidgen) + vgcreate ceph-vg-${random_uuid} ${bl_device} + vg_name=$(get_vg_name_from_device ${bl_device}) + vgrename ceph-vg-${random_uuid} ${vg_name} + echo "OSD Init: Created volume group ${vg_name} for device ${bl_device}." + fi + RESULTING_VG=${vg_name} +} + +function create_lv_if_needed { + local bl_device=$1 + local vg_name=$2 + local options=$3 + local lv_name=${4:-$(get_lv_name_from_device ${bl_device} lv)} + + if [[ ! "$(lvdisplay | awk '/LV Name/{print $3}' | grep ${lv_name})" ]]; then + lvcreate ${options} -n ${lv_name} ${vg_name} + echo "OSD Init: Created logical volume ${lv_name} in group ${vg_name} for device ${bl_device}." + fi + RESULTING_LV=${vg_name}/${lv_name} +} + +function osd_disk_prechecks { + if [[ -z "${OSD_DEVICE}" ]]; then + echo "ERROR- You must provide a device to build your OSD ie: /dev/sdb" + exit 1 + fi + + if [[ ! -b "${OSD_DEVICE}" ]]; then + echo "ERROR- The device pointed by OSD_DEVICE (${OSD_DEVICE}) doesn't exist !" + exit 1 + fi + + if [ ! -e ${OSD_BOOTSTRAP_KEYRING} ]; then + echo "ERROR- ${OSD_BOOTSTRAP_KEYRING} must exist. You can extract it from your current monitor by running 'ceph auth get client.bootstrap-osd -o ${OSD_BOOTSTRAP_KEYRING}'" + exit 1 + fi + + timeout 10 ceph --name client.bootstrap-osd --keyring ${OSD_BOOTSTRAP_KEYRING} health || exit 1 +} + +function perform_zap { + if [[ ${ZAP_EXTRA_PARTITIONS} != "" ]]; then + # This used for filestore/blockstore only + echo "OSD Init: Zapping extra partitions ${ZAP_EXTRA_PARTITIONS}" + zap_extra_partitions "${ZAP_EXTRA_PARTITIONS}" + fi + echo "OSD Init: Zapping device ${OSD_DEVICE}..." + disk_zap ${OSD_DEVICE} + DISK_ZAPPED=1 + udev_settle +} + + +####################################################################### +# Main program +####################################################################### + +if [[ "${STORAGE_TYPE}" != "directory" ]]; then + + # Check to make sure we have what we need to continue + osd_disk_prechecks + + # Settle LVM changes before inspecting volumes + udev_settle + + # Rename VGs first + if [[ "${OSD_DEVICE}" ]]; then + OSD_DEVICE=$(readlink -f ${OSD_DEVICE}) + rename_vg ${OSD_DEVICE} + fi + + # Rename block DB device VG next + if [[ "${BLOCK_DB}" ]]; then + BLOCK_DB=$(readlink -f ${BLOCK_DB}) + rename_vg ${BLOCK_DB} + fi + + # Rename block WAL device VG next + if [[ "${BLOCK_WAL}" ]]; then + BLOCK_WAL=$(readlink -f ${BLOCK_WAL}) + rename_vg ${BLOCK_WAL} + fi + + # Rename LVs after VGs are correct + rename_lvs ${OSD_DEVICE} + + # Update tags (all VG and LV names should be correct before calling this) + update_lv_tags ${OSD_DEVICE} + + # Settle LVM changes again after any changes have been made + udev_settle + + # Initialize some important global variables + CEPH_LVM_PREPARE=1 + OSD_ID=$(get_osd_id_from_device ${OSD_DEVICE}) + DISK_ZAPPED=0 + ZAP_DEVICE=0 + ZAP_EXTRA_PARTITIONS="" + + # The disk may need to be zapped or some LVs may need to be deleted before + # moving on with the disk preparation. + determine_what_needs_zapping + + if [[ ${ZAP_DEVICE} -eq 1 ]]; then + perform_zap + fi + + # Prepare the disk for use + osd_disk_prepare + + # Clean up resources held by the common script + common_cleanup +fi diff --git a/ceph-osd/templates/bin/utils/_checkDNS.sh.tpl b/ceph-osd/templates/bin/utils/_checkDNS.sh.tpl new file mode 100644 index 0000000000..b7e360b2fe --- /dev/null +++ b/ceph-osd/templates/bin/utils/_checkDNS.sh.tpl @@ -0,0 +1,38 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +: "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}" +ENDPOINT="{$1}" + +function check_mon_dns () { + GREP_CMD=$(grep -rl 'ceph-mon' ${CEPH_CONF}) + + if [[ "${ENDPOINT}" == "{up}" ]]; then + echo "If DNS is working, we are good here" + elif [[ "${ENDPOINT}" != "" ]]; then + if [[ ${GREP_CMD} != "" ]]; then + # No DNS, write CEPH MONs IPs into ${CEPH_CONF} + sh -c -e "cat ${CEPH_CONF}.template | sed 's/mon_host.*/mon_host = ${ENDPOINT}/g' | tee ${CEPH_CONF}" > /dev/null 2>&1 + else + echo "endpoints are already cached in ${CEPH_CONF}" + exit + fi + fi +} + +check_mon_dns + +exit diff --git a/ceph-osd/templates/bin/utils/_defragOSDs.sh.tpl b/ceph-osd/templates/bin/utils/_defragOSDs.sh.tpl new file mode 100644 index 0000000000..18920a0ff7 --- /dev/null +++ b/ceph-osd/templates/bin/utils/_defragOSDs.sh.tpl @@ -0,0 +1,41 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +source /tmp/utils-resolveLocations.sh + +if [ "x${STORAGE_TYPE%-*}" == "xblock" ]; then + OSD_DEVICE=$(readlink -f ${STORAGE_LOCATION}) + ODEV=$(echo ${OSD_DEVICE} | sed 's/[0-9]//g' | cut -f 3 -d '/') + OSD_PATH=$(cat /proc/mounts | awk '/ceph-/{print $2}') + OSD_STORE=$(cat ${OSD_PATH}/type) + DATA_PART=$(cat /proc/mounts | awk '/ceph-/{print $1}') + + ODEV_ROTATIONAL=$(cat /sys/block/${ODEV}/queue/rotational) + ODEV_SCHEDULER=$(cat /sys/block/${ODEV}/queue/scheduler | tr -d '[]') + + # NOTE(supamatt): TODO implement bluestore defrag options once it's available upstream + if [ "${ODEV_ROTATIONAL}" -eq "1" ] && [ "x${OSD_STORE}" == "xfilestore" ]; then + # NOTE(supamatt): Switch to CFQ in order to not block I/O + echo "cfq" | tee /sys/block/${ODEV}/queue/scheduler || true + ionice -c 3 xfs_fsr "${OSD_DEVICE}" 2>/dev/null + # NOTE(supamatt): Switch back to previous IO scheduler + echo ${ODEV_SCHEDULER} | tee /sys/block/${ODEV}/queue/scheduler || true + fi +fi + +exit 0 diff --git a/ceph-osd/templates/bin/utils/_resolveLocations.sh.tpl b/ceph-osd/templates/bin/utils/_resolveLocations.sh.tpl new file mode 100644 index 0000000000..f36afa2d1a --- /dev/null +++ b/ceph-osd/templates/bin/utils/_resolveLocations.sh.tpl @@ -0,0 +1,41 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +if [[ "${STORAGE_LOCATION}" ]]; then + STORAGE_LOCATION=$(ls ${STORAGE_LOCATION}) + if [[ `echo "${STORAGE_LOCATION}" | wc -w` -ge 2 ]]; then + echo "ERROR- Multiple locations found: ${STORAGE_LOCATION}" + exit 1 + fi +fi + +if [[ "${BLOCK_DB}" ]]; then + BLOCK_DB=$(ls ${BLOCK_DB}) + if [[ `echo "${BLOCK_DB}" | wc -w` -ge 2 ]]; then + echo "ERROR- Multiple locations found: ${BLOCK_DB}" + exit 1 + fi +fi + +if [[ "${BLOCK_WAL}" ]]; then + BLOCK_WAL=$(ls ${BLOCK_WAL}) + if [[ `echo "${BLOCK_WAL}" | wc -w` -ge 2 ]]; then + echo "ERROR- Multiple locations found: ${BLOCK_WAL}" + exit 1 + fi +fi diff --git a/ceph-osd/templates/configmap-bin.yaml b/ceph-osd/templates/configmap-bin.yaml new file mode 100644 index 0000000000..adb6a09851 --- /dev/null +++ b/ceph-osd/templates/configmap-bin.yaml @@ -0,0 +1,71 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_bin }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ printf "%s-%s" $envAll.Release.Name "bin" | quote }} +data: +{{- if .Values.images.local_registry.active }} + image-repo-sync.sh: | +{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} +{{- end }} +{{- if .Values.bootstrap.enabled }} + bootstrap.sh: | +{{ tuple "bin/_bootstrap.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} + post-apply.sh: | +{{ tuple "bin/_post-apply.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + osd-start.sh: | +{{ tuple "bin/osd/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + log-tail.sh: | +{{ tuple "bin/osd/_log-tail.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + osd-directory-ceph-volume.sh: | +{{ tuple "bin/osd/_directory.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + osd-block-ceph-volume.sh: | +{{ tuple "bin/osd/ceph-volume/_block.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + osd-bluestore-ceph-volume.sh: | +{{ tuple "bin/osd/ceph-volume/_bluestore.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + osd-init-ceph-volume-helper-bluestore.sh: | +{{ tuple "bin/osd/ceph-volume/_init-ceph-volume-helper-bluestore.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + osd-init-ceph-volume-helper-directory.sh: | +{{ tuple "bin/osd/ceph-volume/_init-ceph-volume-helper-directory.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + osd-init-ceph-volume-helper-block-logical.sh: | +{{ tuple "bin/osd/ceph-volume/_init-ceph-volume-helper-block-logical.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + osd-init-ceph-volume.sh: | +{{ tuple "bin/osd/ceph-volume/_init-with-ceph-volume.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + osd-common-ceph-volume.sh: | +{{ tuple "bin/osd/ceph-volume/_common.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + osd-init.sh: | +{{ tuple "bin/osd/_init.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + osd-check.sh: | +{{ tuple "bin/osd/_check.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + osd-stop.sh: | +{{ tuple "bin/osd/_stop.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + log-runner-stop.sh: | +{{ tuple "bin/osd/_log-runner-stop.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + init-dirs.sh: | +{{ tuple "bin/_init-dirs.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + helm-tests.sh: | +{{ tuple "bin/_helm-tests.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + utils-checkDNS.sh: | +{{ tuple "bin/utils/_checkDNS.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + utils-defragOSDs.sh: | +{{ tuple "bin/utils/_defragOSDs.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + utils-resolveLocations.sh: | +{{ tuple "bin/utils/_resolveLocations.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} diff --git a/ceph-osd/templates/configmap-etc.yaml b/ceph-osd/templates/configmap-etc.yaml new file mode 100644 index 0000000000..3e4c9c00f4 --- /dev/null +++ b/ceph-osd/templates/configmap-etc.yaml @@ -0,0 +1,50 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "ceph.osd.configmap.etc" }} +{{- $configMapName := index . 0 }} +{{- $envAll := index . 1 }} +{{- with $envAll }} + +{{- if empty .Values.conf.ceph.global.mon_host -}} +{{- $monHost := tuple "ceph_mon" "internal" "mon_msgr2" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} +{{- $_ := $monHost | set .Values.conf.ceph.global "mon_host" -}} +{{- end -}} + +{{- if empty .Values.conf.ceph.global.fsid -}} +{{- $_ := uuidv4 | set .Values.conf.ceph.global "fsid" -}} +{{- end -}} + +{{- if empty .Values.conf.ceph.osd.cluster_network -}} +{{- $_ := .Values.network.cluster | set .Values.conf.ceph.osd "cluster_network" -}} +{{- end -}} + +{{- if empty .Values.conf.ceph.osd.public_network -}} +{{- $_ := .Values.network.public | set .Values.conf.ceph.osd "public_network" -}} +{{- end -}} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $configMapName }} +data: + ceph.conf: | +{{ include "helm-toolkit.utils.to_ini" .Values.conf.ceph | indent 4 }} + storage.json: | +{{ toPrettyJson .Values.conf.storage | indent 4 }} +{{- end }} +{{- end }} +{{- if .Values.manifests.configmap_etc }} +{{- list (printf "%s-%s" .Release.Name "etc") . | include "ceph.osd.configmap.etc" }} +{{- end }} diff --git a/ceph-osd/templates/daemonset-osd.yaml b/ceph-osd/templates/daemonset-osd.yaml new file mode 100644 index 0000000000..565f00a79c --- /dev/null +++ b/ceph-osd/templates/daemonset-osd.yaml @@ -0,0 +1,540 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "osdLivenessProbeTemplate" -}} +exec: + command: + - /tmp/osd-check.sh +{{- end -}} + +{{- define "osdReadinessProbeTemplate" -}} +exec: + command: + - /tmp/osd-check.sh +{{- end -}} + +{{- if .Values.manifests.daemonset_osd }} +{{- $envAll := . }} + +{{- $serviceAccountName := (printf "%s" .Release.Name) }} +{{ tuple . "osd" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ $serviceAccountName }} +rules: + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ $serviceAccountName }} + apiGroup: rbac.authorization.k8s.io +{{- end }} + +{{- define "ceph.osd.daemonset" }} +{{- $daemonset := index . 0 }} +{{- $configMapName := index . 1 }} +{{- $serviceAccountName := index . 2 }} +{{- $envAll := index . 3 }} +{{- with $envAll }} +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: ceph-osd + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "ceph" "osd" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + selector: + matchLabels: +{{ tuple $envAll "ceph" "osd" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} +{{ tuple $envAll "osd" | include "helm-toolkit.snippets.kubernetes_upgrades_daemonset" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "ceph" "osd" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} +{{ dict "envAll" $envAll "podName" "ceph-osd-default" "containerNames" (list "ceph-osd-default" "log-runner" "ceph-init-dirs" "ceph-log-ownership" "osd-init" "init" ) | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "osd" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + nodeSelector: + {{ .Values.labels.osd.node_selector_key }}: {{ .Values.labels.osd.node_selector_value }} + hostNetwork: true + hostPID: true + hostIPC: true + dnsPolicy: {{ .Values.pod.dns_policy }} + initContainers: +{{ tuple $envAll "osd" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + - name: ceph-init-dirs +{{ tuple $envAll "ceph_osd" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ dict "envAll" $envAll "application" "osd" "container" "ceph_init_dirs" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/init-dirs.sh + env: + # NOTE(portdirect): These environment variables will be populated + # dynamicly at the point of render. + # - name: JOURNAL_LOCATION + # value: /var/lib/openstack-helm/ceph/osd/journal-one + # - name: STORAGE_LOCATION + # value: /var/lib/openstack-helm/ceph/osd/data-one + # - name: JOURNAL_TYPE + # value: directory + # - name: STORAGE_TYPE + # value: directory + - name: CLUSTER + value: "ceph" + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: MON_PORT + value: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: MON_PORT_V2 + value: {{ tuple "ceph_mon" "internal" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: pod-run + mountPath: /run + - name: pod-etc-ceph + mountPath: /etc/ceph + - name: ceph-osd-bin + mountPath: /tmp/init-dirs.sh + subPath: init-dirs.sh + readOnly: true + - name: ceph-osd-etc + mountPath: /etc/ceph/storage.json + subPath: storage.json + readOnly: true + - name: pod-var-lib-ceph + mountPath: /var/lib/ceph + readOnly: false + - name: pod-var-lib-ceph-crash + mountPath: /var/lib/ceph/crash + readOnly: false + - name: pod-var-lib-ceph-tmp + mountPath: /var/lib/ceph/tmp + readOnly: false + - name: pod-var-crash + mountPath: /var/crash + mountPropagation: HostToContainer + readOnly: false + - name: ceph-log-ownership +{{ tuple $envAll "ceph_osd" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ dict "envAll" $envAll "application" "osd" "container" "ceph_log_ownership" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + env: + # NOTE(portdirect): These environment variables will be populated + # dynamicly at the point of render and added to all containers in the + # pod + # - name: JOURNAL_LOCATION + # value: /var/lib/openstack-helm/ceph/osd/journal-one + # - name: STORAGE_LOCATION + # value: /var/lib/openstack-helm/ceph/osd/data-one + # - name: JOURNAL_TYPE + # value: directory + # - name: STORAGE_TYPE + # value: directory + - name: CLUSTER + value: "ceph" + - name: CEPH_GET_ADMIN_KEY + value: "1" + command: + - chown + - -R + - ceph:root + - /var/log/ceph + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: pod-run + mountPath: /run + - name: pod-etc-ceph + mountPath: /etc/ceph + - name: pod-var-log + mountPath: /var/log/ceph + readOnly: false + - name: osd-init +{{ tuple $envAll "ceph_osd" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.osd | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "osd" "container" "osd_init" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + env: + # NOTE(portdirect): These environment variables will be populated + # dynamicly at the point of render and added to all containers in the + # pod + # - name: JOURNAL_LOCATION + # value: /var/lib/openstack-helm/ceph/osd/journal-one + # - name: STORAGE_LOCATION + # value: /var/lib/openstack-helm/ceph/osd/data-one + # - name: JOURNAL_TYPE + # value: directory + # - name: STORAGE_TYPE + # value: directory + - name: CLUSTER + value: "ceph" + - name: DEPLOY_TOOL + value: {{ .Values.deploy.tool }} + - name: OSD_FORCE_REPAIR + value: {{ .Values.deploy.osd_force_repair | quote }} + - name: CEPH_GET_ADMIN_KEY + value: "1" + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: MON_PORT + value: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: MON_PORT_V2 + value: {{ tuple "ceph_mon" "internal" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + command: + - /tmp/osd-init.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: pod-run + mountPath: /run + - name: pod-etc-ceph + mountPath: /etc/ceph + - name: ceph-osd-bin + mountPath: /tmp/osd-init.sh + subPath: osd-init.sh + readOnly: true + - name: ceph-osd-bin + mountPath: /tmp/init-ceph-volume-helper-bluestore.sh + subPath: osd-init-ceph-volume-helper-bluestore.sh + readOnly: true + - name: ceph-osd-bin + mountPath: /tmp/init-ceph-volume-helper-directory.sh + subPath: osd-init-ceph-volume-helper-directory.sh + readOnly: true + - name: ceph-osd-bin + mountPath: /tmp/init-ceph-volume-helper-block-logical.sh + subPath: osd-init-ceph-volume-helper-block-logical.sh + readOnly: true + - name: ceph-osd-bin + mountPath: /tmp/init-ceph-volume.sh + subPath: osd-init-ceph-volume.sh + readOnly: true + - name: ceph-osd-bin + mountPath: /tmp/osd-common-ceph-volume.sh + subPath: osd-common-ceph-volume.sh + readOnly: true + - name: ceph-osd-bin + mountPath: /tmp/utils-resolveLocations.sh + subPath: utils-resolveLocations.sh + readOnly: true + - name: ceph-osd-etc + mountPath: /etc/ceph/ceph.conf.template + subPath: ceph.conf + readOnly: true + - name: ceph-osd-etc + mountPath: /etc/ceph/storage.json + subPath: storage.json + readOnly: true + - name: ceph-bootstrap-osd-keyring + mountPath: /var/lib/ceph/bootstrap-osd/ceph.keyring + subPath: ceph.keyring + readOnly: false + - name: devices + mountPath: /dev + readOnly: false + - name: pod-var-lib-ceph + mountPath: /var/lib/ceph + readOnly: false + - name: pod-var-lib-ceph-crash + mountPath: /var/lib/ceph/crash + readOnly: false + - name: pod-var-lib-ceph-tmp + mountPath: /var/lib/ceph/tmp + readOnly: false + - name: run-lvm + mountPath: /run/lvm + readOnly: false + - name: run-udev + mountPath: /run/udev + readOnly: false + - name: pod-etc-lvm + mountPath: /etc/lvm + readOnly: false + - name: data + mountPath: /var/lib/ceph/osd + readOnly: false + - name: journal + mountPath: /var/lib/ceph/journal + readOnly: false + - name: pod-var-log + mountPath: /var/log/ceph + readOnly: false + - name: pod-var-crash + mountPath: /var/crash + mountPropagation: HostToContainer + readOnly: false + containers: + - name: log-runner +{{ tuple $envAll "ceph_osd" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ dict "envAll" $envAll "application" "osd" "container" "log_runner" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + env: + - name: DAEMON_NAME + value: "ceph-osd" + - name: TRUNCATE_SIZE + value: {{ .Values.logging.truncate.size | quote }} + - name: TRUNCATE_PERIOD + value: {{ .Values.logging.truncate.period | quote }} + - name: WAIT_FOR_OSD_ID_TIMEOUT + value: {{ .Values.logging.osd_id.timeout | quote }} + command: + - /tmp/log-tail.sh + lifecycle: + preStop: + exec: + command: + - /tmp/log-runner-stop.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: ceph-osd-bin + mountPath: /tmp/log-tail.sh + subPath: log-tail.sh + readOnly: true + - name: pod-var-log + mountPath: /var/log/ceph + readOnly: false + - name: ceph-osd-bin + mountPath: /tmp/log-runner-stop.sh + subPath: log-runner-stop.sh + readOnly: true + - name: ceph-osd-default +{{ tuple $envAll "ceph_osd" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.osd | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "osd" "container" "osd_pod" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + env: + # NOTE(portdirect): These environment variables will be populated + # dynamicly at the point of render. + # - name: JOURNAL_LOCATION + # value: /var/lib/openstack-helm/ceph/osd/journal-one + # - name: STORAGE_LOCATION + # value: /var/lib/openstack-helm/ceph/osd/data-one + # - name: JOURNAL_TYPE + # value: directory + # - name: STORAGE_TYPE + # value: directory + - name: CLUSTER + value: "ceph" + - name: DEPLOY_TOOL + value: {{ .Values.deploy.tool }} + - name: CEPH_GET_ADMIN_KEY + value: "1" + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: MON_PORT + value: {{ tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: MON_PORT_V2 + value: {{ tuple "ceph_mon" "internal" "mon_msgr2" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + command: + - /tmp/osd-start.sh + lifecycle: + preStop: + exec: + command: + - /tmp/osd-stop.sh +{{ dict "envAll" . "component" "ceph-osd" "container" "ceph-osd" "type" "liveness" "probeTemplate" (include "osdLivenessProbeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | trim | indent 10 }} +{{ dict "envAll" . "component" "ceph-osd" "container" "ceph-osd" "type" "readiness" "probeTemplate" (include "osdReadinessProbeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | trim | indent 10 }} + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: pod-run + mountPath: /run + - name: pod-etc-ceph + mountPath: /etc/ceph + - name: pod-forego + mountPath: /etc/forego + - name: ceph-osd-bin + mountPath: /tmp/osd-start.sh + subPath: osd-start.sh + readOnly: true + - name: ceph-osd-bin + mountPath: /tmp/osd-directory-ceph-volume.sh + subPath: osd-directory-ceph-volume.sh + readOnly: true + - name: ceph-osd-bin + mountPath: /tmp/osd-block-ceph-volume.sh + subPath: osd-block-ceph-volume.sh + readOnly: true + - name: ceph-osd-bin + mountPath: /tmp/osd-bluestore-ceph-volume.sh + subPath: osd-bluestore-ceph-volume.sh + readOnly: true + - name: ceph-osd-bin + mountPath: /tmp/osd-check.sh + subPath: osd-check.sh + readOnly: true + - name: ceph-osd-bin + mountPath: /tmp/osd-stop.sh + subPath: osd-stop.sh + readOnly: true + - name: ceph-osd-bin + mountPath: /tmp/utils-checkDNS.sh + subPath: utils-checkDNS.sh + readOnly: true + - name: ceph-osd-bin + mountPath: /tmp/osd-common-ceph-volume.sh + subPath: osd-common-ceph-volume.sh + readOnly: true + - name: ceph-osd-bin + mountPath: /tmp/utils-resolveLocations.sh + subPath: utils-resolveLocations.sh + readOnly: true + - name: ceph-osd-bin + mountPath: /tmp/utils-defragOSDs.sh + subPath: utils-defragOSDs.sh + readOnly: true + - name: ceph-osd-etc + mountPath: /etc/ceph/storage.json + subPath: storage.json + readOnly: true + - name: ceph-osd-etc + mountPath: /etc/ceph/ceph.conf.template + subPath: ceph.conf + readOnly: true + - name: ceph-bootstrap-osd-keyring + mountPath: /var/lib/ceph/bootstrap-osd/ceph.keyring + subPath: ceph.keyring + readOnly: false + - name: devices + mountPath: /dev + readOnly: false + - name: pod-var-lib-ceph + mountPath: /var/lib/ceph + readOnly: false + - name: pod-var-lib-ceph-crash + mountPath: /var/lib/ceph/crash + readOnly: false + - name: pod-var-lib-ceph-tmp + mountPath: /var/lib/ceph/tmp + readOnly: false + - name: run-lvm + mountPath: /run/lvm + readOnly: false + - name: run-udev + mountPath: /run/udev + readOnly: false + - name: pod-etc-lvm + mountPath: /etc/lvm + readOnly: false + - name: data + mountPath: /var/lib/ceph/osd + readOnly: false + - name: journal + mountPath: /var/lib/ceph/journal + readOnly: false + - name: pod-var-log + mountPath: /var/log/ceph + readOnly: false + - name: pod-var-crash + mountPath: /var/crash + mountPropagation: HostToContainer + readOnly: false + volumes: + - name: pod-tmp + emptyDir: {} + - name: pod-run + emptyDir: + medium: "Memory" + - name: pod-etc-ceph + emptyDir: {} + - name: pod-forego + emptyDir: {} + - name: devices + hostPath: + path: /dev + - name: run-lvm + hostPath: + path: /run/lvm + - name: run-udev + hostPath: + path: /run/udev + - name: pod-etc-lvm + emptyDir: {} + - name: pod-var-lib-ceph + emptyDir: {} + - name: pod-var-lib-ceph-crash + hostPath: + path: /var/lib/openstack-helm/ceph/crash + type: DirectoryOrCreate + - name: pod-var-lib-ceph-tmp + hostPath: + path: /var/lib/openstack-helm/ceph/var-tmp + type: DirectoryOrCreate + - name: pod-var-crash + hostPath: + path: /var/crash + type: DirectoryOrCreate + - name: pod-var-log + emptyDir: {} + - name: ceph-osd-bin + configMap: + name: {{ printf "%s-%s" $envAll.Release.Name "bin" | quote }} + defaultMode: 0555 + - name: ceph-osd-etc + configMap: + name: {{ $configMapName }} + defaultMode: 0444 + - name: ceph-bootstrap-osd-keyring + secret: + secretName: {{ .Values.secrets.keyrings.osd }} + # NOTE(portdirect): If directory mounts are to be used for OSD's + # they will automaticly be inserted here, with the format: + # - name: data + # hostPath: + # path: /var/lib/foo + # - name: journal + # hostPath: + # path: /var/lib/bar + +{{- end }} +{{- end }} + +{{- if .Values.manifests.daemonset_osd }} +{{- $daemonset := .Values.daemonset.prefix_name }} +{{- $configMapName := (printf "%s-%s" .Release.Name "etc") }} +{{- $serviceAccountName := (printf "%s" .Release.Name) }} +{{- $daemonset_yaml := list $daemonset $configMapName $serviceAccountName . | include "ceph.osd.daemonset" | toString | fromYaml }} +{{- $configmap_yaml := "ceph.osd.configmap.etc" }} +{{- list $daemonset $daemonset_yaml $configmap_yaml $configMapName . | include "ceph.utils.osd_daemonset_overrides" }} +{{- end }} diff --git a/ceph-osd/templates/job-bootstrap.yaml b/ceph-osd/templates/job-bootstrap.yaml new file mode 100644 index 0000000000..eb1c01900d --- /dev/null +++ b/ceph-osd/templates/job-bootstrap.yaml @@ -0,0 +1,82 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_bootstrap .Values.bootstrap.enabled }} +{{- $envAll := . }} + +{{- $serviceAccountName := "ceph-osd-bootstrap" }} +{{ tuple $envAll "bootstrap" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: ceph-osd-bootstrap + labels: +{{ tuple $envAll "ceph" "bootstrap" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +spec: + template: + metadata: + labels: +{{ tuple $envAll "ceph" "bootstrap" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "bootstrap" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} + initContainers: +{{ tuple $envAll "bootstrap" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: ceph-osd-bootstrap +{{ tuple $envAll "ceph_bootstrap" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.bootstrap | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "bootstrap" "container" "ceph_osd_bootstrap" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/bootstrap.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: pod-etc-ceph + mountPath: /etc/ceph + - name: ceph-osd-bin + mountPath: /tmp/bootstrap.sh + subPath: bootstrap.sh + readOnly: true + - name: ceph-osd-etc + mountPath: /etc/ceph/ceph.conf + subPath: ceph.conf + readOnly: true + - name: ceph-osd-admin-keyring + mountPath: /etc/ceph/ceph.client.admin.keyring + subPath: ceph.client.admin.keyring + readOnly: true + volumes: + - name: pod-tmp + emptyDir: {} + - name: pod-etc-ceph + emptyDir: {} + - name: ceph-osd-bin + configMap: + name: {{ printf "%s-%s" $envAll.Release.Name "bin" | quote }} + defaultMode: 0555 + - name: ceph-osd-etc + configMap: + name: {{ printf "%s-%s" $envAll.Release.Name "etc" | quote }} + defaultMode: 0444 + - name: ceph-osd-admin-keyring + secret: + secretName: {{ .Values.secrets.keyrings.admin }} +{{- end }} diff --git a/ceph-osd/templates/job-image-repo-sync.yaml b/ceph-osd/templates/job-image-repo-sync.yaml new file mode 100644 index 0000000000..54ffc6627d --- /dev/null +++ b/ceph-osd/templates/job-image-repo-sync.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "ceph-osd" -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/ceph-osd/templates/job-post-apply.yaml b/ceph-osd/templates/job-post-apply.yaml new file mode 100644 index 0000000000..393769d950 --- /dev/null +++ b/ceph-osd/templates/job-post-apply.yaml @@ -0,0 +1,149 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if eq .Values.pod.lifecycle.upgrades.daemonsets.pod_replacement_strategy "OnDelete" }} +{{- if and .Values.manifests.job_post_apply }} +{{- $envAll := . }} + +{{- $serviceAccountName := printf "%s-%s" .Release.Name "post-apply" }} +{{ tuple $envAll "post-apply" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ $serviceAccountName }} +rules: + - apiGroups: + - '' + resources: + - pods + - events + - jobs + - pods/exec + verbs: + - create + - get + - delete + - list + - apiGroups: + - 'apps' + resources: + - daemonsets + verbs: + - get + - list + - apiGroups: + - 'batch' + resources: + - jobs + verbs: + - get + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ $serviceAccountName }} + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ $serviceAccountName }} + labels: +{{ tuple $envAll "ceph-upgrade" "post-apply" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +spec: + template: + metadata: + labels: +{{ tuple $envAll "ceph-upgrade" "post-apply" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} +{{ dict "envAll" $envAll "podName" "ceph-osd-post-apply" "containerNames" (list "ceph-osd-post-apply" "init" ) | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "post_apply" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} + initContainers: +{{ tuple $envAll "post-apply" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: ceph-osd-post-apply +{{ tuple $envAll "ceph_config_helper" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.bootstrap | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "post_apply" "container" "ceph_osd_post_apply" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + env: + - name: CLUSTER + value: "ceph" + - name: CEPH_NAMESPACE + value: {{ .Release.Namespace }} + - name: RELEASE_GROUP_NAME + value: {{ .Release.Name }} + - name: REQUIRED_PERCENT_OF_OSDS + value: {{ .Values.conf.ceph.target.required_percent_of_osds | ceil | quote }} + - name: DISRUPTIVE_OSD_RESTART + value: {{ .Values.conf.storage.disruptive_osd_restart | quote }} + - name: UNCONDITIONAL_OSD_RESTART + value: {{ .Values.conf.storage.unconditional_osd_restart | quote }} + command: + - /tmp/post-apply.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: pod-etc-ceph + mountPath: /etc/ceph + - name: ceph-osd-bin + mountPath: /tmp/post-apply.sh + subPath: post-apply.sh + readOnly: true + - name: ceph-osd-bin + mountPath: /tmp/wait-for-pods.sh + subPath: wait-for-pods.sh + readOnly: true + - name: ceph-osd-etc + mountPath: /etc/ceph/ceph.conf + subPath: ceph.conf + readOnly: true + - name: ceph-osd-admin-keyring + mountPath: /etc/ceph/ceph.client.admin.keyring + subPath: ceph.client.admin.keyring + readOnly: true + volumes: + - name: pod-tmp + emptyDir: {} + - name: pod-etc-ceph + emptyDir: {} + - name: ceph-osd-bin + configMap: + name: {{ printf "%s-%s" $envAll.Release.Name "bin" | quote }} + defaultMode: 0555 + - name: ceph-osd-etc + configMap: + name: {{ printf "%s-%s" $envAll.Release.Name "etc" | quote }} + defaultMode: 0444 + - name: ceph-osd-admin-keyring + secret: + secretName: {{ .Values.secrets.keyrings.admin }} +{{- end }} +{{- end }} diff --git a/ceph-osd/templates/pod-helm-tests.yaml b/ceph-osd/templates/pod-helm-tests.yaml new file mode 100644 index 0000000000..9a5c98b8cc --- /dev/null +++ b/ceph-osd/templates/pod-helm-tests.yaml @@ -0,0 +1,85 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.helm_tests }} +{{- $envAll := . }} + +{{- $serviceAccountName := printf "%s-%s" $envAll.Release.Name "test" }} +{{ tuple $envAll "tests" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: v1 +kind: Pod +metadata: + name: {{ $serviceAccountName }} + labels: +{{ tuple $envAll "ceph-osd" "test" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: + "helm.sh/hook": test-success +{{ dict "envAll" $envAll "podName" "ceph-osd-test" "containerNames" (list "init" "ceph-cluster-helm-test") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 4 }} +spec: +{{ dict "envAll" $envAll "application" "test" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 2 }} + restartPolicy: Never + serviceAccountName: {{ $serviceAccountName }} + nodeSelector: + {{ .Values.labels.test.node_selector_key }}: {{ .Values.labels.test.node_selector_value }} + initContainers: +{{ tuple $envAll "tests" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 4 }} + containers: + - name: ceph-cluster-helm-test +{{ tuple $envAll "ceph_config_helper" | include "helm-toolkit.snippets.image" | indent 6 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.tests | include "helm-toolkit.snippets.kubernetes_resources" | indent 6 }} +{{ dict "envAll" $envAll "application" "test" "container" "ceph_cluster_helm_test" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 6 }} + env: + - name: CLUSTER + value: "ceph" + - name: CEPH_DEPLOYMENT_NAMESPACE + value: {{ .Release.Namespace }} + - name: REQUIRED_PERCENT_OF_OSDS + value: {{ .Values.conf.ceph.target.required_percent_of_osds | ceil | quote }} + command: + - /tmp/helm-tests.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: pod-etc-ceph + mountPath: /etc/ceph + - name: ceph-osd-bin + mountPath: /tmp/helm-tests.sh + subPath: helm-tests.sh + readOnly: true + - name: ceph-client-admin-keyring + mountPath: /etc/ceph/ceph.client.admin.keyring + subPath: ceph.client.admin.keyring + readOnly: true + - name: ceph-osd-etc + mountPath: /etc/ceph/ceph.conf + subPath: ceph.conf + readOnly: true + volumes: + - name: pod-tmp + emptyDir: {} + - name: pod-etc-ceph + emptyDir: {} + - name: ceph-osd-bin + configMap: + name: {{ printf "%s-%s" $envAll.Release.Name "bin" | quote }} + defaultMode: 0555 + - name: ceph-client-admin-keyring + secret: + secretName: {{ .Values.secrets.keyrings.admin }} + - name: ceph-osd-etc + configMap: + name: {{ printf "%s-%s" $envAll.Release.Name "etc" | quote }} + defaultMode: 0444 +{{- end }} diff --git a/ceph-osd/templates/secret-registry.yaml b/ceph-osd/templates/secret-registry.yaml new file mode 100644 index 0000000000..da979b3223 --- /dev/null +++ b/ceph-osd/templates/secret-registry.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_registry .Values.endpoints.oci_image_registry.auth.enabled }} +{{ include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) }} +{{- end }} diff --git a/ceph-osd/templates/utils/_osd_daemonset_overrides.tpl b/ceph-osd/templates/utils/_osd_daemonset_overrides.tpl new file mode 100644 index 0000000000..e152666341 --- /dev/null +++ b/ceph-osd/templates/utils/_osd_daemonset_overrides.tpl @@ -0,0 +1,390 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "ceph.utils.match_exprs_hash" }} + {{- $match_exprs := index . 0 }} + {{- $context := index . 1 }} + {{- $_ := set $context.Values "__match_exprs_hash_content" "" }} + {{- range $match_expr := $match_exprs }} + {{- $_ := set $context.Values "__match_exprs_hash_content" (print $context.Values.__match_exprs_hash_content $match_expr.key $match_expr.operator ($match_expr.values | quote)) }} + {{- end }} + {{- $context.Values.__match_exprs_hash_content | sha256sum | trunc 8 }} + {{- $_ := unset $context.Values "__match_exprs_hash_content" }} +{{- end }} + +{{- define "ceph.utils.osd_daemonset_overrides" }} + {{- $daemonset := index . 0 }} + {{- $daemonset_yaml := index . 1 }} + {{- $configmap_include := index . 2 }} + {{- $configmap_name := index . 3 }} + {{- $context := index . 4 }} + {{- $_ := unset $context ".Files" }} + {{- $_ := set $context.Values "__daemonset_yaml" $daemonset_yaml }} + {{- $daemonset_root_name := printf "ceph_%s" $daemonset }} + {{- $_ := set $context.Values "__daemonset_list" list }} + {{- $_ := set $context.Values "__default" dict }} + {{- if hasKey $context.Values.conf "overrides" }} + {{- range $key, $val := $context.Values.conf.overrides }} + + {{- if eq $key $daemonset_root_name }} + {{- range $type, $type_data := . }} + + {{- if eq $type "hosts" }} + {{- range $host_data := . }} + {{/* dictionary that will contain all info needed to generate this + iteration of the daemonset */}} + {{- $current_dict := dict }} + + {{/* set daemonset name */}} + {{- $_ := set $current_dict "name" $host_data.name }} + + {{/* apply overrides */}} + {{- $override_conf_copy := $host_data.conf }} + {{/* Deep copy to prevent https://storyboard.openstack.org/#!/story/2005936 */}} + {{- $root_conf_copy := omit ($context.Values.conf | toYaml | fromYaml) "overrides" }} + {{- $merged_dict := mergeOverwrite $root_conf_copy $override_conf_copy }} + {{- $root_conf_copy2 := dict "conf" $merged_dict }} + {{- $context_values := omit (omit ($context.Values | toYaml | fromYaml) "conf") "__daemonset_list" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2 }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $current_dict "nodeData" $root_conf_copy4 }} + + {{/* Schedule to this host explicitly. */}} + {{- $nodeSelector_dict := dict }} + + {{- $_ := set $nodeSelector_dict "key" "kubernetes.io/hostname" }} + {{- $_ := set $nodeSelector_dict "operator" "In" }} + + {{- $values_list := list $host_data.name }} + {{- $_ := set $nodeSelector_dict "values" $values_list }} + + {{- $list_aggregate := list $nodeSelector_dict }} + {{- $_ := set $current_dict "matchExpressions" $list_aggregate }} + + {{/* store completed daemonset entry/info into global list */}} + {{- $list_aggregate := append $context.Values.__daemonset_list $current_dict }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + + {{- end }} + {{- end }} + + {{- if eq $type "labels" }} + {{- $_ := set $context.Values "__label_list" . }} + {{- range $label_data := . }} + {{/* dictionary that will contain all info needed to generate this + iteration of the daemonset. */}} + {{- $_ := set $context.Values "__current_label" dict }} + + {{/* set daemonset name */}} + {{- $_ := set $context.Values.__current_label "name" $label_data.label.key }} + + {{/* apply overrides */}} + {{- $override_conf_copy := $label_data.conf }} + {{/* Deep copy to prevent https://storyboard.openstack.org/#!/story/2005936 */}} + {{- $root_conf_copy := omit ($context.Values.conf | toYaml | fromYaml) "overrides" }} + {{- $merged_dict := mergeOverwrite $root_conf_copy $override_conf_copy }} + {{- $root_conf_copy2 := dict "conf" $merged_dict }} + {{- $context_values := omit (omit ($context.Values | toYaml | fromYaml) "conf") "__daemonset_list" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2 }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__current_label "nodeData" $root_conf_copy4 }} + + {{/* Schedule to the provided label value(s) */}} + {{- $label_dict := omit $label_data.label "NULL" }} + {{- $_ := set $label_dict "operator" "In" }} + {{- $list_aggregate := list $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + + {{/* Do not schedule to other specified labels, with higher + precedence as the list position increases. Last defined label + is highest priority. */}} + {{- $other_labels := without $context.Values.__label_list $label_data }} + {{- range $label_data2 := $other_labels }} + {{- $label_dict := omit $label_data2.label "NULL" }} + + {{- $_ := set $label_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__current_label.matchExpressions $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + {{- end }} + {{- $_ := set $context.Values "__label_list" $other_labels }} + + {{/* Do not schedule to any other specified hosts */}} + {{- range $type, $type_data := $val }} + {{- if eq $type "hosts" }} + {{- range $host_data := . }} + {{- $label_dict := dict }} + + {{- $_ := set $label_dict "key" "kubernetes.io/hostname" }} + {{- $_ := set $label_dict "operator" "NotIn" }} + + {{- $values_list := list $host_data.name }} + {{- $_ := set $label_dict "values" $values_list }} + + {{- $list_aggregate := append $context.Values.__current_label.matchExpressions $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + {{- end }} + {{- end }} + {{- end }} + + {{/* store completed daemonset entry/info into global list */}} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__current_label }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + {{- $_ := unset $context.Values "__current_label" }} + + {{- end }} + {{- end }} + {{- end }} + + {{/* scheduler exceptions for the default daemonset */}} + {{- $_ := set $context.Values.__default "matchExpressions" list }} + + {{- range $type, $type_data := . }} + {{/* Do not schedule to other specified labels */}} + {{- if eq $type "labels" }} + {{- range $label_data := . }} + {{- $default_dict := omit $label_data.label "NULL" }} + + {{- $_ := set $default_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__default.matchExpressions $default_dict }} + {{- $_ := set $context.Values.__default "matchExpressions" $list_aggregate }} + {{- end }} + {{- end }} + {{/* Do not schedule to other specified hosts */}} + {{- if eq $type "hosts" }} + {{- range $host_data := . }} + {{- $default_dict := dict }} + + {{- $_ := set $default_dict "key" "kubernetes.io/hostname" }} + {{- $_ := set $default_dict "operator" "NotIn" }} + + {{- $values_list := list $host_data.name }} + {{- $_ := set $default_dict "values" $values_list }} + + {{- $list_aggregate := append $context.Values.__default.matchExpressions $default_dict }} + {{- $_ := set $context.Values.__default "matchExpressions" $list_aggregate }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + + {{/* generate the default daemonset */}} + + {{/* set name */}} + {{- $_ := set $context.Values.__default "name" "default" }} + + {{/* no overrides apply, so copy as-is */}} + {{- $root_conf_copy1 := omit $context.Values.conf "overrides" }} + {{- $root_conf_copy2 := dict "conf" $root_conf_copy1 }} + {{- $context_values := omit $context.Values "conf" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2 }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__default "nodeData" $root_conf_copy4 }} + + {{/* add to global list */}} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__default }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + + {{- $_ := set $context.Values "__last_configmap_name" $configmap_name }} + {{- range $current_dict := $context.Values.__daemonset_list }} + + {{- $context_novalues := omit $context "Values" }} + {{- $merged_dict := mergeOverwrite $context_novalues $current_dict.nodeData }} + {{- $_ := set $current_dict "nodeData" $merged_dict }} + + {{/* name needs to be a DNS-1123 compliant name. Ensure lower case */}} + {{- $name_format1 := printf (print $daemonset_root_name "-" $current_dict.name) | lower }} + {{/* labels may contain underscores which would be invalid here, so we replace them with dashes + there may be other valid label names which would make for an invalid DNS-1123 name + but these will be easier to handle in future with sprig regex* functions + (not availabile in helm 2.5.1) */}} + {{- $name_format2 := $name_format1 | replace "_" "-" | replace "." "-" }} + {{/* To account for the case where the same label is defined multiple times in overrides + (but with different label values), we add a sha of the scheduling data to ensure + name uniqueness */}} + {{- $_ := set $current_dict "dns_1123_name" dict }} + {{- if hasKey $current_dict "matchExpressions" }} + {{- $_ := set $current_dict "dns_1123_name" (printf (print $name_format2 "-" (list $current_dict.matchExpressions $context | include "ceph.utils.match_exprs_hash"))) }} + {{- else }} + {{- $_ := set $current_dict "dns_1123_name" $name_format2 }} + {{- end }} + + {{/* set daemonset metadata name */}} + {{- if not $context.Values.__daemonset_yaml.metadata }}{{- $_ := set $context.Values.__daemonset_yaml "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.metadata.name }}{{- $_ := set $context.Values.__daemonset_yaml.metadata "name" dict }}{{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.metadata "name" $current_dict.dns_1123_name }} + + {{/* cross-reference configmap name to container volume definitions */}} + {{- $_ := set $context.Values "__volume_list" list }} + {{- range $current_volume := $context.Values.__daemonset_yaml.spec.template.spec.volumes }} + {{- $_ := set $context.Values "__volume" $current_volume }} + {{- if hasKey $context.Values.__volume "configMap" }} + {{- if eq $context.Values.__volume.configMap.name $context.Values.__last_configmap_name }} + {{- $_ := set $context.Values.__volume.configMap "name" $current_dict.dns_1123_name }} + {{- end }} + {{- end }} + {{- $updated_list := append $context.Values.__volume_list $context.Values.__volume }} + {{- $_ := set $context.Values "__volume_list" $updated_list }} + {{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "volumes" $context.Values.__volume_list }} + + {{/* populate scheduling restrictions */}} + {{- if hasKey $current_dict "matchExpressions" }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "affinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity "nodeAffinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity "requiredDuringSchedulingIgnoredDuringExecution" dict }}{{- end }} + {{- $match_exprs := dict }} + {{- $_ := set $match_exprs "matchExpressions" $current_dict.matchExpressions }} + {{- $appended_match_expr := list $match_exprs }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" $appended_match_expr }} + {{- end }} + + {{/* input value hash for current set of values overrides */}} + {{- if not $context.Values.__daemonset_yaml.spec }}{{- $_ := set $context.Values.__daemonset_yaml "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template }}{{- $_ := set $context.Values.__daemonset_yaml.spec "template" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata.annotations }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata "annotations" dict }}{{- end }} + {{- $cmap := list $current_dict.dns_1123_name $current_dict.nodeData | include $configmap_include }} + {{- $values_hash := $cmap | quote | sha256sum }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-etc-hash" $values_hash }} + + {{/* generate configmap */}} +--- +{{ $cmap }} + + {{/* generate daemonset yaml */}} +{{ range $k, $v := index $current_dict.nodeData.Values.conf.storage "osd" }} +--- +{{- $_ := set $context.Values "__tmpYAML" dict }} + +{{ $dsNodeName := index $context.Values.__daemonset_yaml.metadata "name" }} +{{ $localDsNodeName := print (trunc 54 $current_dict.dns_1123_name) "-" (print $dsNodeName $k | quote | sha256sum | trunc 8) }} +{{- if not $context.Values.__tmpYAML.metadata }}{{- $_ := set $context.Values.__tmpYAML "metadata" dict }}{{- end }} +{{- $_ := set $context.Values.__tmpYAML.metadata "name" $localDsNodeName }} + +{{ $podDataVols := index $context.Values.__daemonset_yaml.spec.template.spec "volumes" }} +{{- $_ := set $context.Values "__tmpPodVols" $podDataVols }} + + {{ if eq $v.data.type "directory" }} + {{ $dataDirVolume := dict "hostPath" (dict "path" $v.data.location) "name" "data" }} + {{ $newPodDataVols := append $context.Values.__tmpPodVols $dataDirVolume }} + {{- $_ := set $context.Values "__tmpPodVols" $newPodDataVols }} + {{ else }} + {{ $dataDirVolume := dict "emptyDir" dict "name" "data" }} + {{ $newPodDataVols := append $context.Values.__tmpPodVols $dataDirVolume }} + {{- $_ := set $context.Values "__tmpPodVols" $newPodDataVols }} + {{ end }} + + {{- if ne $v.data.type "bluestore" }} + {{ if eq $v.journal.type "directory" }} + {{ $journalDirVolume := dict "hostPath" (dict "path" $v.journal.location) "name" "journal" }} + {{ $newPodDataVols := append $context.Values.__tmpPodVols $journalDirVolume }} + {{- $_ := set $context.Values "__tmpPodVols" $newPodDataVols }} + {{ else }} + {{ $dataDirVolume := dict "emptyDir" dict "name" "journal" }} + {{ $newPodDataVols := append $context.Values.__tmpPodVols $dataDirVolume }} + {{- $_ := set $context.Values "__tmpPodVols" $newPodDataVols }} + {{ end }} + {{ else }} + {{ $dataDirVolume := dict "emptyDir" dict "name" "journal" }} + {{ $newPodDataVols := append $context.Values.__tmpPodVols $dataDirVolume }} + {{- $_ := set $context.Values "__tmpPodVols" $newPodDataVols }} + {{- end }} + + {{- if not $context.Values.__tmpYAML.spec }}{{- $_ := set $context.Values.__tmpYAML "spec" dict }}{{- end }} + {{- if not $context.Values.__tmpYAML.spec.template }}{{- $_ := set $context.Values.__tmpYAML.spec "template" dict }}{{- end }} + {{- if not $context.Values.__tmpYAML.spec.template.spec }}{{- $_ := set $context.Values.__tmpYAML.spec.template "spec" dict }}{{- end }} + {{- $_ := set $context.Values.__tmpYAML.spec.template.spec "volumes" $context.Values.__tmpPodVols }} + + {{- if not $context.Values.__tmpYAML.spec }}{{- $_ := set $context.Values.__tmpYAML "spec" dict }}{{- end }} + {{- if not $context.Values.__tmpYAML.spec.template }}{{- $_ := set $context.Values.__tmpYAML.spec "template" dict }}{{- end }} + {{- if not $context.Values.__tmpYAML.spec.template.spec }}{{- $_ := set $context.Values.__tmpYAML.spec.template "spec" dict }}{{- end }} + {{- if not $context.Values.__tmpYAML.spec.template.spec.containers }}{{- $_ := set $context.Values.__tmpYAML.spec.template.spec "containers" list }}{{- end }} + {{- if not $context.Values.__tmpYAML.spec.template.spec.initContainers }}{{- $_ := set $context.Values.__tmpYAML.spec.template.spec "initContainers" list }}{{- end }} + + {{- $_ := set $context.Values "__tmpYAMLcontainers" list }} + {{- range $podContainer := $context.Values.__daemonset_yaml.spec.template.spec.containers }} + {{- $_ := set $context.Values "_tmpYAMLcontainer" $podContainer }} + {{- if empty $context.Values._tmpYAMLcontainer.env }} + {{- $_ := set $context.Values._tmpYAMLcontainer "env" ( list ) }} + {{- end }} + {{- $tmpcontainerEnv := omit $context.Values._tmpYAMLcontainer "env" }} + {{- if eq $v.data.type "bluestore" }} + {{- if and $v.block_db $v.block_wal}} + {{ $containerEnv := prepend (prepend (prepend ( prepend ( prepend ( prepend (index $context.Values._tmpYAMLcontainer "env") (dict "name" "STORAGE_TYPE" "value" $v.data.type)) (dict "name" "STORAGE_LOCATION" "value" $v.data.location)) (dict "name" "BLOCK_DB" "value" $v.block_db.location)) (dict "name" "BLOCK_DB_SIZE" "value" $v.block_db.size)) (dict "name" "BLOCK_WAL" "value" $v.block_wal.location)) (dict "name" "BLOCK_WAL_SIZE" "value" $v.block_wal.size) }} + {{- $_ := set $tmpcontainerEnv "env" $containerEnv }} + {{- else if $v.block_db }} + {{ $containerEnv := prepend (prepend ( prepend ( prepend (index $context.Values._tmpYAMLcontainer "env") (dict "name" "STORAGE_TYPE" "value" $v.data.type)) (dict "name" "STORAGE_LOCATION" "value" $v.data.location)) (dict "name" "BLOCK_DB" "value" $v.block_db.location)) (dict "name" "BLOCK_DB_SIZE" "value" $v.block_db.size) }} + {{- $_ := set $tmpcontainerEnv "env" $containerEnv }} + {{- else if $v.block_wal }} + {{ $containerEnv := prepend (prepend ( prepend ( prepend (index $context.Values._tmpYAMLcontainer "env") (dict "name" "STORAGE_TYPE" "value" $v.data.type)) (dict "name" "STORAGE_LOCATION" "value" $v.data.location)) (dict "name" "BLOCK_WAL" "value" $v.block_wal.location)) (dict "name" "BLOCK_WAL_SIZE" "value" $v.block_wal.size) }} + {{- $_ := set $tmpcontainerEnv "env" $containerEnv }} + {{ else }} + {{ $containerEnv := prepend (prepend (index $context.Values._tmpYAMLcontainer "env") (dict "name" "STORAGE_TYPE" "value" $v.data.type)) (dict "name" "STORAGE_LOCATION" "value" $v.data.location) }} + {{- $_ := set $tmpcontainerEnv "env" $containerEnv }} + {{- end }} + {{ else }} + {{ $containerEnv := prepend (prepend (prepend ( prepend (index $context.Values._tmpYAMLcontainer "env") (dict "name" "STORAGE_TYPE" "value" $v.data.type)) (dict "name" "JOURNAL_TYPE" "value" $v.journal.type)) (dict "name" "STORAGE_LOCATION" "value" $v.data.location)) (dict "name" "JOURNAL_LOCATION" "value" $v.journal.location) }} + {{- $_ := set $tmpcontainerEnv "env" $containerEnv }} + {{- end }} + {{- $localInitContainerEnv := omit $context.Values._tmpYAMLcontainer "env" }} + {{- $_ := set $localInitContainerEnv "env" $tmpcontainerEnv.env }} + {{ $containerList := append $context.Values.__tmpYAMLcontainers $localInitContainerEnv }} + {{ $_ := set $context.Values "__tmpYAMLcontainers" $containerList }} + {{ end }} + {{- $_ := set $context.Values.__tmpYAML.spec.template.spec "containers" $context.Values.__tmpYAMLcontainers }} + + {{- $_ := set $context.Values "__tmpYAMLinitContainers" list }} + {{- range $podContainer := $context.Values.__daemonset_yaml.spec.template.spec.initContainers }} + {{- $_ := set $context.Values "_tmpYAMLinitContainer" $podContainer }} + {{- $tmpinitcontainerEnv := omit $context.Values._tmpYAMLinitContainer "env" }} + {{- if eq $v.data.type "bluestore" }} + {{- if and $v.block_db $v.block_wal}} + {{ $initcontainerEnv := prepend (prepend (prepend ( prepend ( prepend ( prepend (index $context.Values._tmpYAMLinitContainer "env") (dict "name" "STORAGE_TYPE" "value" $v.data.type)) (dict "name" "STORAGE_LOCATION" "value" $v.data.location)) (dict "name" "BLOCK_DB" "value" $v.block_db.location)) (dict "name" "BLOCK_DB_SIZE" "value" $v.block_db.size)) (dict "name" "BLOCK_WAL" "value" $v.block_wal.location)) (dict "name" "BLOCK_WAL_SIZE" "value" $v.block_wal.size) }} + {{- $_ := set $tmpinitcontainerEnv "env" $initcontainerEnv }} + {{- else if $v.block_db }} + {{ $initcontainerEnv := prepend (prepend ( prepend ( prepend (index $context.Values._tmpYAMLinitContainer "env") (dict "name" "STORAGE_TYPE" "value" $v.data.type)) (dict "name" "STORAGE_LOCATION" "value" $v.data.location)) (dict "name" "BLOCK_DB" "value" $v.block_db.location)) (dict "name" "BLOCK_DB_SIZE" "value" $v.block_db.size) }} + {{- $_ := set $tmpinitcontainerEnv "env" $initcontainerEnv }} + {{- else if $v.block_wal }} + {{ $initcontainerEnv := prepend (prepend ( prepend ( prepend (index $context.Values._tmpYAMLinitContainer "env") (dict "name" "STORAGE_TYPE" "value" $v.data.type)) (dict "name" "STORAGE_LOCATION" "value" $v.data.location)) (dict "name" "BLOCK_WAL" "value" $v.block_wal.location)) (dict "name" "BLOCK_WAL_SIZE" "value" $v.block_wal.size) }} + {{- $_ := set $tmpinitcontainerEnv "env" $initcontainerEnv }} + {{ else }} + {{ $initcontainerEnv := prepend (prepend (index $context.Values._tmpYAMLinitContainer "env") (dict "name" "STORAGE_TYPE" "value" $v.data.type)) (dict "name" "STORAGE_LOCATION" "value" $v.data.location) }} + {{- $_ := set $tmpinitcontainerEnv "env" $initcontainerEnv }} + {{- end }} + {{ else }} + {{ $initcontainerEnv := prepend (prepend (prepend ( prepend (index $context.Values._tmpYAMLinitContainer "env") (dict "name" "STORAGE_TYPE" "value" $v.data.type)) (dict "name" "JOURNAL_TYPE" "value" $v.journal.type)) (dict "name" "STORAGE_LOCATION" "value" $v.data.location)) (dict "name" "JOURNAL_LOCATION" "value" $v.journal.location) }} + {{- $_ := set $tmpinitcontainerEnv "env" $initcontainerEnv }} + {{- end }} + {{- $localInitContainerEnv := omit $context.Values._tmpYAMLinitContainer "env" }} + {{- $_ := set $localInitContainerEnv "env" $tmpinitcontainerEnv.env }} + {{ $initContainerList := append $context.Values.__tmpYAMLinitContainers $localInitContainerEnv }} + {{ $_ := set $context.Values "__tmpYAMLinitContainers" $initContainerList }} + {{ end }} + {{- $_ := set $context.Values.__tmpYAML.spec.template.spec "initContainers" $context.Values.__tmpYAMLinitContainers }} + + {{- $_ := set $context.Values.__tmpYAML.spec.template.spec "volumes" $context.Values.__tmpPodVols }} + +{{ merge $context.Values.__tmpYAML $context.Values.__daemonset_yaml | toYaml }} + +{{ end }} + +--- + {{- $_ := set $context.Values "__last_configmap_name" $current_dict.dns_1123_name }} + {{- end }} +{{- end }} diff --git a/ceph-osd/values.yaml b/ceph-osd/values.yaml new file mode 100644 index 0000000000..69a5fb3682 --- /dev/null +++ b/ceph-osd/values.yaml @@ -0,0 +1,431 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for ceph-osd. +# This is a YAML-formatted file. +# Declare name/value pairs to be passed into your templates. +# name: value + +--- +images: + pull_policy: IfNotPresent + tags: + ceph_osd: 'docker.io/openstackhelm/ceph-daemon:ubuntu_jammy_19.2.1-1-20250207' + ceph_bootstrap: 'docker.io/openstackhelm/ceph-daemon:ubuntu_jammy_19.2.1-1-20250207' + ceph_config_helper: 'docker.io/openstackhelm/ceph-config-helper:ubuntu_jammy_19.2.1-1-20250207' + dep_check: 'quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal' + image_repo_sync: 'docker.io/library/docker:17.07.0' + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +labels: + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + test: + node_selector_key: openstack-control-plane + node_selector_value: enabled + osd: + node_selector_key: ceph-osd + node_selector_value: enabled + +# The default deploy tool is ceph-volume. "ceph-disk" was finally removed as it +# had been deprecated from Nautilus and was not being used. +deploy: + tool: "ceph-volume" +# NOTE: set this to 1 if osd disk needs wiping in case of reusing from previous deployment + osd_force_repair: 1 + +pod: + security_context: + osd: + pod: + runAsUser: 65534 + container: + ceph_init_dirs: + runAsUser: 0 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + ceph_log_ownership: + runAsUser: 0 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + osd_init: + runAsUser: 0 + privileged: true + readOnlyRootFilesystem: true + osd_pod: + runAsUser: 0 + privileged: true + readOnlyRootFilesystem: true + log_runner: + # run as "ceph" user + runAsUser: 64045 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + bootstrap: + pod: + runAsUser: 65534 + container: + ceph_osd_bootstrap: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + post_apply: + pod: + runAsUser: 65534 + container: + ceph_osd_post_apply: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + test: + pod: + runAsUser: 65534 + container: + ceph_cluster_helm_test: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + dns_policy: "ClusterFirstWithHostNet" + lifecycle: + upgrades: + daemonsets: + pod_replacement_strategy: RollingUpdate + osd: + enabled: true + min_ready_seconds: 0 + max_unavailable: 1 + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + weight: + default: 10 + resources: + enabled: false + osd: + requests: + memory: "2Gi" + cpu: "1000m" + limits: + memory: "5Gi" + cpu: "2000m" + tests: + requests: + memory: "10Mi" + cpu: "250m" + limits: + memory: "50Mi" + cpu: "500m" + jobs: + image_repo_sync: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + probes: + ceph-osd: + ceph-osd: + readiness: + enabled: true + params: + initialDelaySeconds: 60 + periodSeconds: 60 + timeoutSeconds: 5 + liveness: + enabled: true + params: + initialDelaySeconds: 120 + periodSeconds: 60 + timeoutSeconds: 5 + +secrets: + keyrings: + osd: ceph-bootstrap-osd-keyring + admin: ceph-client-admin-keyring + oci_image_registry: + ceph-osd: ceph-osh-oci-image-registry-key + +network: + public: 192.168.0.0/16 + cluster: 192.168.0.0/16 + +jobs: + ceph_defragosds: + # Execute the 1st of each month + cron: "0 0 1 * *" + history: + # Number of successful job to keep + successJob: 1 + # Number of failed job to keep + failJob: 1 + concurrency: + # Skip new job if previous job still active + execPolicy: Forbid + startingDeadlineSecs: 60 + +conf: + ceph: + global: + # auth + cephx: true + cephx_require_signatures: false + cephx_cluster_require_signatures: true + cephx_service_require_signatures: false + objecter_inflight_op_bytes: "1073741824" + objecter_inflight_ops: 10240 + debug_ms: "0/0" + mon_osd_down_out_interval: 1800 + mon_osd_down_out_subtree_limit: root + mon_osd_min_in_ratio: 0 + mon_osd_min_up_ratio: 0 + osd: + osd_mkfs_type: xfs + osd_mkfs_options_xfs: -f -i size=2048 + osd_max_object_name_len: 256 + ms_bind_port_min: 6800 + ms_bind_port_max: 7100 + osd_snap_trim_priority: 1 + osd_snap_trim_sleep: 0.1 + osd_pg_max_concurrent_snap_trims: 1 + filestore_merge_threshold: -10 + filestore_split_multiple: 12 + filestore_max_sync_interval: 10 + osd_scrub_begin_hour: 22 + osd_scrub_end_hour: 4 + osd_scrub_during_recovery: false + osd_scrub_sleep: 0.1 + osd_scrub_chunk_min: 1 + osd_scrub_chunk_max: 4 + osd_scrub_load_threshold: 10.0 + osd_deep_scrub_stride: "1048576" + osd_scrub_priority: 1 + osd_recovery_op_priority: 1 + osd_recovery_max_active: 1 + osd_mount_options_xfs: "rw,noatime,largeio,inode64,swalloc,logbufs=8,logbsize=256k,allocsize=4M" + osd_journal_size: 10240 + osd_crush_update_on_start: false + target: + # This is just for helm tests to proceed the deployment if we have mentioned % of + # osds are up and running. + required_percent_of_osds: 75 + + storage: + # NOTE(supamatt): By default use host based buckets for failure domains. Any `failure_domain` defined must + # match the failure domain used on your CRUSH rules for pools. For example with a crush rule of + # rack_replicated_rule you would specify "rack" as the `failure_domain` to use. + # `failure_domain`: Set the CRUSH bucket type for your OSD to reside in. See the supported CRUSH configuration + # as listed here: Supported CRUSH configuration is listed here: http://docs.ceph.com/docs/nautilus/rados/operations/crush-map/ + # if failure domain is rack then it will check for node label "rack" and get the value from it to create the rack, if there + # is no label rack then it will use following options. + # `failure_domain_by_hostname`: Specify the portion of the hostname to use for your failure domain bucket name. + # `failure_domain_by_hostname_map`: Explicit mapping of hostname to failure domain, as a simpler alternative to overrides. + # `failure_domain_name`: Manually name the failure domain bucket name. This configuration option should only be used + # when using host based overrides. + failure_domain: "host" + failure_domain_by_hostname: "false" + failure_domain_by_hostname_map: {} + # Example: + # failure_domain_map_hostname_map: + # hostfoo: rack1 + # hostbar: rack1 + # hostbaz: rack2 + # hostqux: rack2 + failure_domain_name: "false" + + # Note: You can override the device class by adding the value (e.g., hdd, ssd or nvme). + # Leave it empty if you don't need to modify the device class. + device_class: "" + + # NOTE(portdirect): for homogeneous clusters the `osd` key can be used to + # define OSD pods that will be deployed across the cluster. + # when specifing whole disk (/dev/sdf) for journals, ceph-osd chart will create + # needed partitions for each OSDs. + osd: + # Below is the current configuration default, which is Bluestore with co-located metadata + # - data: + # type: bluestore + # location: /dev/sdb # Use a valid device here + + # Separate block devices may be used for block.db and/or block.wal + # Specify the location and size in Gb. It is recommended that the + # block_db size isn't smaller than 4% of block. For example, if the + # block size is 1TB, then block_db shouldn't be less than 40GB. + # A size suffix of K for kilobytes, M for megabytes, G for gigabytes, + # T for terabytes, P for petabytes or E for exabytes is optional. + # Default unit is megabytes. + # block_db: + # location: /dev/sdc + # size: "96GB" + # block_wal: + # location: /dev/sdc + # size: "2GB" + + # Block-based Filestore OSDs with separate journal block devices + # - data: + # type: block-logical + # location: /dev/sdd + # journal: + # type: block-logical + # location: /dev/sdf1 + # - data: + # type: block-logical + # location: /dev/sde + # journal: + # type: block-logical + # location: /dev/sdf2 + + # Block-based Filestore OSDs with directory-based journals + # - data: + # type: block-logical + # location: /dev/sdg + # journal: + # type: directory + # location: /var/lib/openstack-helm/ceph/osd/journal-sdg + + # Directory-based Filestore OSD + # - data: + # type: directory + # location: /var/lib/openstack-helm/ceph/osd/osd-one + # journal: + # type: directory + # location: /var/lib/openstack-helm/ceph/osd/journal-one + + # The post-apply job will restart OSDs without disruption by default. Set + # this value to "true" to restart all OSDs at once. This will accomplish + # OSD restarts more quickly with disruption. + disruptive_osd_restart: "false" + + # The post-apply job will try to determine if OSDs need to be restarted and + # only restart them if necessary. Set this value to "true" to restart OSDs + # unconditionally. + unconditional_osd_restart: "false" + +# NOTE(portdirect): for heterogeneous clusters the overrides section can be used to define +# OSD pods that will be deployed upon specifc nodes. +# overrides: +# ceph_osd: +# hosts: +# - name: host1.fqdn +# conf: +# storage: +# failure_domain_name: "rack1" +# osd: +# - data: +# type: directory +# location: /var/lib/openstack-helm/ceph/osd/data-three +# journal: +# type: directory +# location: /var/lib/openstack-helm/ceph/osd/journal-three + +daemonset: + prefix_name: "osd" + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - ceph-osd-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + osd: + jobs: + - ceph-storage-keys-generator + - ceph-osd-keyring-generator + services: + - endpoint: internal + service: ceph_mon + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + tests: + jobs: + - ceph-storage-keys-generator + - ceph-osd-keyring-generator + services: + - endpoint: internal + service: ceph_mon + +logging: + truncate: + size: 0 + period: 3600 + osd_id: + timeout: 300 + +bootstrap: + enabled: true + script: | + ceph -s + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + ceph-osd: + username: ceph-osd + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + ceph_mon: + namespace: null + hosts: + default: ceph-mon + discovery: ceph-mon-discovery + host_fqdn_override: + default: null + port: + mon: + default: 6789 + mon_msgr2: + default: 3300 + +manifests: + configmap_bin: true + configmap_etc: true + configmap_test_bin: true + daemonset_osd: true + job_bootstrap: false + job_post_apply: true + job_image_repo_sync: true + helm_tests: true + secret_registry: true +... diff --git a/ceph-provisioners/Chart.yaml b/ceph-provisioners/Chart.yaml new file mode 100644 index 0000000000..f9207ef3c1 --- /dev/null +++ b/ceph-provisioners/Chart.yaml @@ -0,0 +1,24 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: v1.0.0 +description: OpenStack-Helm Ceph Provisioner +name: ceph-provisioners +version: 2024.2.0 +home: https://github.com/ceph/ceph +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit + version: ">= 0.1.0" +... diff --git a/ceph-provisioners/crds/snapshot.storage.k8s.io_volumesnapshotclasses.yaml b/ceph-provisioners/crds/snapshot.storage.k8s.io_volumesnapshotclasses.yaml new file mode 100644 index 0000000000..4cacd07f68 --- /dev/null +++ b/ceph-provisioners/crds/snapshot.storage.k8s.io_volumesnapshotclasses.yaml @@ -0,0 +1,87 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.8.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/665" + creationTimestamp: null + name: volumesnapshotclasses.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotClass + listKind: VolumeSnapshotClassList + plural: volumesnapshotclasses + shortNames: + - vsclass + - vsclasses + singular: volumesnapshotclass + scope: Cluster + versions: + - additionalPrinterColumns: + - jsonPath: .driver + name: Driver + type: string + - description: Determines whether a VolumeSnapshotContent created through the + VolumeSnapshotClass should be deleted when its bound VolumeSnapshot is deleted. + jsonPath: .deletionPolicy + name: DeletionPolicy + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: VolumeSnapshotClass specifies parameters that a underlying storage + system uses when creating a volume snapshot. A specific VolumeSnapshotClass + is used by specifying its name in a VolumeSnapshot object. VolumeSnapshotClasses + are non-namespaced + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + deletionPolicy: + description: deletionPolicy determines whether a VolumeSnapshotContent + created through the VolumeSnapshotClass should be deleted when its bound + VolumeSnapshot is deleted. Supported values are "Retain" and "Delete". + "Retain" means that the VolumeSnapshotContent and its physical snapshot + on underlying storage system are kept. "Delete" means that the VolumeSnapshotContent + and its physical snapshot on underlying storage system are deleted. + Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the storage driver that handles this + VolumeSnapshotClass. Required. + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + parameters: + additionalProperties: + type: string + description: parameters is a key-value map with storage driver specific + parameters for creating snapshots. These values are opaque to Kubernetes. + type: object + required: + - deletionPolicy + - driver + type: object + served: true + storage: true + subresources: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +... diff --git a/ceph-provisioners/crds/snapshot.storage.k8s.io_volumesnapshotcontents.yaml b/ceph-provisioners/crds/snapshot.storage.k8s.io_volumesnapshotcontents.yaml new file mode 100644 index 0000000000..cafa9b9565 --- /dev/null +++ b/ceph-provisioners/crds/snapshot.storage.k8s.io_volumesnapshotcontents.yaml @@ -0,0 +1,256 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.8.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/665" + creationTimestamp: null + name: volumesnapshotcontents.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshotContent + listKind: VolumeSnapshotContentList + plural: volumesnapshotcontents + shortNames: + - vsc + - vscs + singular: volumesnapshotcontent + scope: Cluster + versions: + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: Represents the complete size of the snapshot in bytes + jsonPath: .status.restoreSize + name: RestoreSize + type: integer + - description: Determines whether this VolumeSnapshotContent and its physical + snapshot on the underlying storage system should be deleted when its bound + VolumeSnapshot is deleted. + jsonPath: .spec.deletionPolicy + name: DeletionPolicy + type: string + - description: Name of the CSI driver used to create the physical snapshot on + the underlying storage system. + jsonPath: .spec.driver + name: Driver + type: string + - description: Name of the VolumeSnapshotClass to which this snapshot belongs. + jsonPath: .spec.volumeSnapshotClassName + name: VolumeSnapshotClass + type: string + - description: Name of the VolumeSnapshot object to which this VolumeSnapshotContent + object is bound. + jsonPath: .spec.volumeSnapshotRef.name + name: VolumeSnapshot + type: string + - description: Namespace of the VolumeSnapshot object to which this VolumeSnapshotContent object is bound. + jsonPath: .spec.volumeSnapshotRef.namespace + name: VolumeSnapshotNamespace + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: VolumeSnapshotContent represents the actual "on-disk" snapshot + object in the underlying storage system + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: spec defines properties of a VolumeSnapshotContent created + by the underlying storage system. Required. + properties: + deletionPolicy: + description: deletionPolicy determines whether this VolumeSnapshotContent + and its physical snapshot on the underlying storage system should + be deleted when its bound VolumeSnapshot is deleted. Supported values + are "Retain" and "Delete". "Retain" means that the VolumeSnapshotContent + and its physical snapshot on underlying storage system are kept. + "Delete" means that the VolumeSnapshotContent and its physical snapshot + on underlying storage system are deleted. For dynamically provisioned + snapshots, this field will automatically be filled in by the CSI + snapshotter sidecar with the "DeletionPolicy" field defined in the + corresponding VolumeSnapshotClass. For pre-existing snapshots, users + MUST specify this field when creating the VolumeSnapshotContent + object. Required. + enum: + - Delete + - Retain + type: string + driver: + description: driver is the name of the CSI driver used to create the + physical snapshot on the underlying storage system. This MUST be + the same as the name returned by the CSI GetPluginName() call for + that driver. Required. + type: string + source: + description: source specifies whether the snapshot is (or should be) + dynamically provisioned or already exists, and just requires a Kubernetes + object representation. This field is immutable after creation. Required. + properties: + snapshotHandle: + description: snapshotHandle specifies the CSI "snapshot_id" of + a pre-existing snapshot on the underlying storage system for + which a Kubernetes object representation was (or should be) + created. This field is immutable. + type: string + volumeHandle: + description: volumeHandle specifies the CSI "volume_id" of the + volume from which a snapshot should be dynamically taken from. + This field is immutable. + type: string + type: object + oneOf: + - required: ["snapshotHandle"] + - required: ["volumeHandle"] + sourceVolumeMode: + description: SourceVolumeMode is the mode of the volume whose snapshot + is taken. Can be either “Filesystem” or “Block”. If not specified, + it indicates the source volume's mode is unknown. This field is + immutable. This field is an alpha field. + type: string + volumeSnapshotClassName: + description: name of the VolumeSnapshotClass from which this snapshot + was (or will be) created. Note that after provisioning, the VolumeSnapshotClass + may be deleted or recreated with different set of values, and as + such, should not be referenced post-snapshot creation. + type: string + volumeSnapshotRef: + description: volumeSnapshotRef specifies the VolumeSnapshot object + to which this VolumeSnapshotContent object is bound. VolumeSnapshot.Spec.VolumeSnapshotContentName + field must reference to this VolumeSnapshotContent's name for the + bidirectional binding to be valid. For a pre-existing VolumeSnapshotContent + object, name and namespace of the VolumeSnapshot object MUST be + provided for binding to happen. This field is immutable after creation. + Required. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: 'If referring to a piece of an object instead of + an entire object, this string should contain a valid JSON/Go + field access statement, such as desiredState.manifest.containers[2]. + For example, if the object reference is to a container within + a pod, this would take on a value like: "spec.containers{name}" + (where "name" refers to the name of the container that triggered + the event) or if no container name is specified "spec.containers[2]" + (container with index 2 in this pod). This syntax is chosen + only to have some well-defined way of referencing a part of + an object. TODO: this design is not final and this field is + subject to change in the future.' + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference + is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids' + type: string + type: object + required: + - deletionPolicy + - driver + - source + - volumeSnapshotRef + type: object + status: + description: status represents the current information of a snapshot. + properties: + creationTime: + description: creationTime is the timestamp when the point-in-time + snapshot is taken by the underlying storage system. In dynamic snapshot + creation case, this field will be filled in by the CSI snapshotter + sidecar with the "creation_time" value returned from CSI "CreateSnapshot" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "creation_time" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. If not specified, it indicates + the creation time is unknown. The format of this field is a Unix + nanoseconds time encoded as an int64. On Unix, the command `date + +%s%N` returns the current time in nanoseconds since 1970-01-01 + 00:00:00 UTC. + format: int64 + type: integer + error: + description: error is the last observed error during snapshot creation, + if any. Upon success after retry, this error field will be cleared. + properties: + message: + description: 'message is a string detailing the encountered error + during snapshot creation if specified. NOTE: message may be + logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if a snapshot is ready to be used + to restore a volume. In dynamic snapshot creation case, this field + will be filled in by the CSI snapshotter sidecar with the "ready_to_use" + value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing + snapshot, this field will be filled with the "ready_to_use" value + returned from the CSI "ListSnapshots" gRPC call if the driver supports + it, otherwise, this field will be set to "True". If not specified, + it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + description: restoreSize represents the complete size of the snapshot + in bytes. In dynamic snapshot creation case, this field will be + filled in by the CSI snapshotter sidecar with the "size_bytes" value + returned from CSI "CreateSnapshot" gRPC call. For a pre-existing + snapshot, this field will be filled with the "size_bytes" value + returned from the CSI "ListSnapshots" gRPC call if the driver supports + it. When restoring a volume from this snapshot, the size of the + volume MUST NOT be smaller than the restoreSize if it is specified, + otherwise the restoration will fail. If not specified, it indicates + that the size is unknown. + format: int64 + minimum: 0 + type: integer + snapshotHandle: + description: snapshotHandle is the CSI "snapshot_id" of a snapshot + on the underlying storage system. If not specified, it indicates + that dynamic snapshot creation has either failed or it is still + in progress. + type: string + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +... diff --git a/ceph-provisioners/crds/snapshot.storage.k8s.io_volumesnapshots.yaml b/ceph-provisioners/crds/snapshot.storage.k8s.io_volumesnapshots.yaml new file mode 100644 index 0000000000..9800e14e8f --- /dev/null +++ b/ceph-provisioners/crds/snapshot.storage.k8s.io_volumesnapshots.yaml @@ -0,0 +1,205 @@ +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.8.0 + api-approved.kubernetes.io: "https://github.com/kubernetes-csi/external-snapshotter/pull/665" + creationTimestamp: null + name: volumesnapshots.snapshot.storage.k8s.io +spec: + group: snapshot.storage.k8s.io + names: + kind: VolumeSnapshot + listKind: VolumeSnapshotList + plural: volumesnapshots + shortNames: + - vs + singular: volumesnapshot + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Indicates if the snapshot is ready to be used to restore a volume. + jsonPath: .status.readyToUse + name: ReadyToUse + type: boolean + - description: If a new snapshot needs to be created, this contains the name of + the source PVC from which this snapshot was (or will be) created. + jsonPath: .spec.source.persistentVolumeClaimName + name: SourcePVC + type: string + - description: If a snapshot already exists, this contains the name of the existing + VolumeSnapshotContent object representing the existing snapshot. + jsonPath: .spec.source.volumeSnapshotContentName + name: SourceSnapshotContent + type: string + - description: Represents the minimum size of volume required to rehydrate from + this snapshot. + jsonPath: .status.restoreSize + name: RestoreSize + type: string + - description: The name of the VolumeSnapshotClass requested by the VolumeSnapshot. + jsonPath: .spec.volumeSnapshotClassName + name: SnapshotClass + type: string + - description: Name of the VolumeSnapshotContent object to which the VolumeSnapshot + object intends to bind to. Please note that verification of binding actually + requires checking both VolumeSnapshot and VolumeSnapshotContent to ensure + both are pointing at each other. Binding MUST be verified prior to usage of + this object. + jsonPath: .status.boundVolumeSnapshotContentName + name: SnapshotContent + type: string + - description: Timestamp when the point-in-time snapshot was taken by the underlying + storage system. + jsonPath: .status.creationTime + name: CreationTime + type: date + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1 + schema: + openAPIV3Schema: + description: VolumeSnapshot is a user's request for either creating a point-in-time + snapshot of a persistent volume, or binding to a pre-existing snapshot. + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + spec: + description: 'spec defines the desired characteristics of a snapshot requested + by a user. More info: https://kubernetes.io/docs/concepts/storage/volume-snapshots#volumesnapshots + Required.' + properties: + source: + description: source specifies where a snapshot will be created from. + This field is immutable after creation. Required. + properties: + persistentVolumeClaimName: + description: persistentVolumeClaimName specifies the name of the + PersistentVolumeClaim object representing the volume from which + a snapshot should be created. This PVC is assumed to be in the + same namespace as the VolumeSnapshot object. This field should + be set if the snapshot does not exists, and needs to be created. + This field is immutable. + type: string + volumeSnapshotContentName: + description: volumeSnapshotContentName specifies the name of a + pre-existing VolumeSnapshotContent object representing an existing + volume snapshot. This field should be set if the snapshot already + exists and only needs a representation in Kubernetes. This field + is immutable. + type: string + type: object + oneOf: + - required: ["persistentVolumeClaimName"] + - required: ["volumeSnapshotContentName"] + volumeSnapshotClassName: + description: 'VolumeSnapshotClassName is the name of the VolumeSnapshotClass + requested by the VolumeSnapshot. VolumeSnapshotClassName may be + left nil to indicate that the default SnapshotClass should be used. + A given cluster may have multiple default Volume SnapshotClasses: + one default per CSI Driver. If a VolumeSnapshot does not specify + a SnapshotClass, VolumeSnapshotSource will be checked to figure + out what the associated CSI Driver is, and the default VolumeSnapshotClass + associated with that CSI Driver will be used. If more than one VolumeSnapshotClass + exist for a given CSI Driver and more than one have been marked + as default, CreateSnapshot will fail and generate an event. Empty + string is not allowed for this field.' + type: string + required: + - source + type: object + status: + description: status represents the current information of a snapshot. + Consumers must verify binding between VolumeSnapshot and VolumeSnapshotContent + objects is successful (by validating that both VolumeSnapshot and VolumeSnapshotContent + point at each other) before using this object. + properties: + boundVolumeSnapshotContentName: + description: 'boundVolumeSnapshotContentName is the name of the VolumeSnapshotContent + object to which this VolumeSnapshot object intends to bind to. If + not specified, it indicates that the VolumeSnapshot object has not + been successfully bound to a VolumeSnapshotContent object yet. NOTE: + To avoid possible security issues, consumers must verify binding + between VolumeSnapshot and VolumeSnapshotContent objects is successful + (by validating that both VolumeSnapshot and VolumeSnapshotContent + point at each other) before using this object.' + type: string + creationTime: + description: creationTime is the timestamp when the point-in-time + snapshot is taken by the underlying storage system. In dynamic snapshot + creation case, this field will be filled in by the snapshot controller + with the "creation_time" value returned from CSI "CreateSnapshot" + gRPC call. For a pre-existing snapshot, this field will be filled + with the "creation_time" value returned from the CSI "ListSnapshots" + gRPC call if the driver supports it. If not specified, it may indicate + that the creation time of the snapshot is unknown. + format: date-time + type: string + error: + description: error is the last observed error during snapshot creation, + if any. This field could be helpful to upper level controllers(i.e., + application controller) to decide whether they should continue on + waiting for the snapshot to be created based on the type of error + reported. The snapshot controller will keep retrying when an error + occurs during the snapshot creation. Upon success, this error field + will be cleared. + properties: + message: + description: 'message is a string detailing the encountered error + during snapshot creation if specified. NOTE: message may be + logged, and it should not contain sensitive information.' + type: string + time: + description: time is the timestamp when the error was encountered. + format: date-time + type: string + type: object + readyToUse: + description: readyToUse indicates if the snapshot is ready to be used + to restore a volume. In dynamic snapshot creation case, this field + will be filled in by the snapshot controller with the "ready_to_use" + value returned from CSI "CreateSnapshot" gRPC call. For a pre-existing + snapshot, this field will be filled with the "ready_to_use" value + returned from the CSI "ListSnapshots" gRPC call if the driver supports + it, otherwise, this field will be set to "True". If not specified, + it means the readiness of a snapshot is unknown. + type: boolean + restoreSize: + type: string + description: restoreSize represents the minimum size of volume required + to create a volume from this snapshot. In dynamic snapshot creation + case, this field will be filled in by the snapshot controller with + the "size_bytes" value returned from CSI "CreateSnapshot" gRPC call. + For a pre-existing snapshot, this field will be filled with the + "size_bytes" value returned from the CSI "ListSnapshots" gRPC call + if the driver supports it. When restoring a volume from this snapshot, + the size of the volume MUST NOT be smaller than the restoreSize + if it is specified, otherwise the restoration will fail. If not + specified, it indicates that the size is unknown. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + required: + - spec + type: object + served: true + storage: true + subresources: + status: {} +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +... diff --git a/ceph-provisioners/templates/bin/_bootstrap.sh.tpl b/ceph-provisioners/templates/bin/_bootstrap.sh.tpl new file mode 100644 index 0000000000..6452d0a073 --- /dev/null +++ b/ceph-provisioners/templates/bin/_bootstrap.sh.tpl @@ -0,0 +1,18 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +{{ .Values.bootstrap.script | default "echo 'Not Enabled'" }} diff --git a/ceph-provisioners/templates/bin/_helm-tests.sh.tpl b/ceph-provisioners/templates/bin/_helm-tests.sh.tpl new file mode 100644 index 0000000000..b22916d5e9 --- /dev/null +++ b/ceph-provisioners/templates/bin/_helm-tests.sh.tpl @@ -0,0 +1,205 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +function reset_test_env() +{ + pvc_namespace=$1 + pod_name=$2 + pvc_name=$3 + echo "--> Resetting POD and PVC before/after test" + if kubectl get pod -n $pvc_namespace $pod_name; then + kubectl delete pod -n $pvc_namespace $pod_name + fi + + if kubectl get cm -n $pvc_namespace ${pod_name}-bin; then + kubectl delete cm -n $pvc_namespace ${pod_name}-bin + fi + + if kubectl get pvc -n $pvc_namespace $pvc_name; then + kubectl delete pvc -n $pvc_namespace $pvc_name; + fi +} + + +function storageclass_validation() +{ + pvc_namespace=$1 + pod_name=$2 + pvc_name=$3 + storageclass=$4 + + echo "--> Starting validation" + + # storageclass check + if ! kubectl get storageclass $storageclass; then + echo "Storageclass: $storageclass is not provisioned." + exit 1 + fi + + tee < Checking RBD storage class." + storageclass={{ $val.metadata.name }} + + storageclass_validation $PVC_NAMESPACE $RBD_TEST_POD_NAME $RBD_TEST_PVC_NAME $storageclass + reset_test_env $PVC_NAMESPACE $RBD_TEST_POD_NAME $RBD_TEST_PVC_NAME +fi + +if [ {{ $val.provisioner }} == "ceph.rbd.csi.ceph.com" ] && [ {{ $val.provision_storage_class }} == true ]; +then + echo "--> Checking CSI RBD storage class." + storageclass={{ $val.metadata.name }} + storageclass_validation $PVC_NAMESPACE $CSI_RBD_TEST_POD_NAME $CSI_RBD_TEST_PVC_NAME $storageclass + reset_test_env $PVC_NAMESPACE $CSI_RBD_TEST_POD_NAME $CSI_RBD_TEST_PVC_NAME +fi + +if [ {{ $val.provisioner }} == "ceph.com/cephfs" ] && [ {{ $val.provision_storage_class }} == true ]; +then + echo "--> Checking cephfs storage class." + storageclass={{ $val.metadata.name }} + storageclass_validation $PVC_NAMESPACE $CEPHFS_TEST_POD_NAME $CEPHFS_TEST_PVC_NAME $storageclass + reset_test_env $PVC_NAMESPACE $CEPHFS_TEST_POD_NAME $CEPHFS_TEST_PVC_NAME +fi +{{- end }} diff --git a/ceph-provisioners/templates/bin/provisioner/cephfs/_client-key-manager.sh.tpl b/ceph-provisioners/templates/bin/provisioner/cephfs/_client-key-manager.sh.tpl new file mode 100644 index 0000000000..421e6f61a3 --- /dev/null +++ b/ceph-provisioners/templates/bin/provisioner/cephfs/_client-key-manager.sh.tpl @@ -0,0 +1,50 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +{{- $envAll := . }} + +CEPH_CEPHFS_KEY=$(kubectl get secret ${PVC_CEPH_CEPHFS_STORAGECLASS_ADMIN_SECRET_NAME} \ + --namespace=${PVC_CEPH_CEPHFS_STORAGECLASS_DEPLOYED_NAMESPACE} \ + -o json ) + +ceph_activate_namespace() { + kube_namespace=$1 + secret_type=$2 + secret_name=$3 + ceph_key=$4 + { + cat <= 0.1.0" +... diff --git a/ceph-rgw/templates/bin/_bootstrap.sh.tpl b/ceph-rgw/templates/bin/_bootstrap.sh.tpl new file mode 100644 index 0000000000..6452d0a073 --- /dev/null +++ b/ceph-rgw/templates/bin/_bootstrap.sh.tpl @@ -0,0 +1,18 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +{{ .Values.bootstrap.script | default "echo 'Not Enabled'" }} diff --git a/ceph-rgw/templates/bin/_ceph-admin-keyring.sh.tpl b/ceph-rgw/templates/bin/_ceph-admin-keyring.sh.tpl new file mode 100644 index 0000000000..adb1bb0073 --- /dev/null +++ b/ceph-rgw/templates/bin/_ceph-admin-keyring.sh.tpl @@ -0,0 +1,25 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +export HOME=/tmp + +cat < /etc/ceph/ceph.client.admin.keyring +[client.admin] + key = $(cat /tmp/client-keyring) +EOF + +exit 0 diff --git a/ceph-rgw/templates/bin/_ceph-rgw-storage-init.sh.tpl b/ceph-rgw/templates/bin/_ceph-rgw-storage-init.sh.tpl new file mode 100644 index 0000000000..7468e90299 --- /dev/null +++ b/ceph-rgw/templates/bin/_ceph-rgw-storage-init.sh.tpl @@ -0,0 +1,57 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -x +if [ "x$STORAGE_BACKEND" == "xceph-rgw" ]; then + SECRET=$(mktemp --suffix .yaml) + KEYRING=$(mktemp --suffix .keyring) + function cleanup { + rm -f ${SECRET} ${KEYRING} + } + trap cleanup EXIT +fi + +function kube_ceph_keyring_gen () { + CEPH_KEY=$1 + CEPH_KEY_TEMPLATE=$2 + sed "s|{{"{{"}} key {{"}}"}}|${CEPH_KEY}|" /tmp/ceph-templates/${CEPH_KEY_TEMPLATE} | base64 -w0 | tr -d '\n' +} + +set -ex +if [ "x$STORAGE_BACKEND" == "xceph-rgw" ]; then + ceph -s + if USERINFO=$(ceph auth get client.bootstrap-rgw); then + KEYSTR=$(echo $USERINFO | sed 's/.*\( key = .*\) caps mon.*/\1/') + echo $KEYSTR > ${KEYRING} + else + #NOTE(Portdirect): Determine proper privs to assign keyring + ceph auth get-or-create client.bootstrap-rgw \ + mon "allow profile bootstrap-rgw" \ + -o ${KEYRING} + fi + FINAL_KEYRING=$(sed -n 's/^[[:blank:]]*key[[:blank:]]\+=[[:blank:]]\(.*\)/\1/p' ${KEYRING}) + cat > ${SECRET} </dev/null || true) +if [[ -z "$RGW_PLACEMENT_TARGET_EXISTS" ]]; then + create_rgw_placement_target "$RGW_ZONEGROUP" "$RGW_PLACEMENT_TARGET" + add_rgw_zone_placement "$RGW_ZONE" "$RGW_PLACEMENT_TARGET" "$RGW_PLACEMENT_TARGET_DATA_POOL" "$RGW_PLACEMENT_TARGET_INDEX_POOL" "$RGW_PLACEMENT_TARGET_DATA_EXTRA_POOL" + RGW_PLACEMENT_TARGET_EXISTS=$(radosgw-admin zonegroup placement get --placement-id "$RGW_PLACEMENT_TARGET" 2>/dev/null || true) +fi +if [[ -n "$RGW_PLACEMENT_TARGET_EXISTS" ]] && + [[ "true" == "$RGW_DELETE_PLACEMENT_TARGET" ]]; then + rm_rgw_zone_placement "$RGW_ZONE" "$RGW_PLACEMENT_TARGET" + delete_rgw_placement_target "$RGW_ZONEGROUP" "$RGW_PLACEMENT_TARGET" +fi +{{- end }} diff --git a/ceph-rgw/templates/bin/_helm-tests.sh.tpl b/ceph-rgw/templates/bin/_helm-tests.sh.tpl new file mode 100644 index 0000000000..cdda9bd150 --- /dev/null +++ b/ceph-rgw/templates/bin/_helm-tests.sh.tpl @@ -0,0 +1,166 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +tmpdir=$(mktemp -d) +declare -a objects_list +total_objects=10 + +#NOTE: This function tests keystone based auth. It uses ceph_config_helper +#container image that has openstack and ceph installed +function rgw_keystone_bucket_validation () +{ + echo "function: rgw_keystone_bucket_validation" + openstack service list + + bucket_stat="$(openstack container list | grep openstack_test_container || true)" + if [[ -n "${bucket_stat}" ]]; then + echo "--> deleting openstack_test_container container" + openstack container delete --recursive openstack_test_container + fi + + echo "--> creating openstack_test_container container" + openstack container create 'openstack_test_container' + + echo "--> list containers" + openstack container list + + bucket_stat="$(openstack container list | grep openstack_test_container || true)" + if [[ -z "${bucket_stat}" ]]; then + echo "--> container openstack_test_container not found" + exit 1 + else + echo "--> container openstack_test_container found" + + for i in $(seq $total_objects); do + openstack object create --name "${objects_list[$i]}" openstack_test_container "${objects_list[$i]}" + echo "--> file ${objects_list[$i]} uploaded to openstack_test_container container" + done + + echo "--> list contents of openstack_test_container container" + openstack object list openstack_test_container + + for i in $(seq $total_objects); do + echo "--> downloading ${objects_list[$i]} object from openstack_test_container container to ${objects_list[$i]}_object${i} file" + openstack object save --file "${objects_list[$i]}_object${i}" openstack_test_container "${objects_list[$i]}" + check_result $? "Error during openstack CLI execution" "The object downloaded successfully" + + echo "--> comparing files: ${objects_list[$i]} and ${objects_list[$i]}_object${i}" + cmp "${objects_list[$i]}" "${objects_list[$i]}_object${i}" + check_result $? "The files are not equal" "The files are equal" + + echo "--> deleting ${objects_list[$i]} object from openstack_test_container container" + openstack object delete openstack_test_container "${objects_list[$i]}" + check_result $? "Error during openstack CLI execution" "The object deleted successfully" + done + + echo "--> deleting openstack_test_container container" + openstack container delete --recursive openstack_test_container + + echo "--> bucket list after deleting container" + openstack container list + fi +} + +#NOTE: This function tests s3 based auto. It uses ceph_rgw container image which has +# s3cmd util install +function rgw_s3_bucket_validation () +{ + echo "function: rgw_s3_bucket_validation" + + bucket=s3://rgw-test-bucket +{{- if .Values.manifests.certificates }} + params="--host=$RGW_HOST --host-bucket=$RGW_HOST --access_key=$S3_ADMIN_ACCESS_KEY --secret_key=$S3_ADMIN_SECRET_KEY --ca-certs=/etc/tls/ca.crt" +{{- else }} + params="--host=$RGW_HOST --host-bucket=$RGW_HOST --access_key=$S3_ADMIN_ACCESS_KEY --secret_key=$S3_ADMIN_SECRET_KEY --no-ssl" +{{- end }} + + bucket_stat="$(s3cmd ls $params | grep ${bucket} || true)" + if [[ -n "${bucket_stat}" ]]; then + s3cmd del --recursive --force $bucket $params + check_result $? "Error during s3cmd execution" "Bucket is deleted" + fi + + s3cmd mb $bucket $params + if [ $? -eq 0 ]; then + echo "Bucket $bucket created" + + for i in $(seq $total_objects); do + s3cmd put "${objects_list[$i]}" $bucket $params + check_result $? "Error during s3cmd execution" "File ${objects_list[$i]##*/} uploaded to bucket" + done + + s3cmd ls $bucket $params + check_result $? "Error during s3cmd execution" "Got list of objects" + + for i in $(seq $total_objects); do + s3cmd get "${bucket}/${objects_list[$i]##*/}" -> "${objects_list[$i]}_s3_object${i}" $params + check_result $? "Error during s3cmd execution" "File ${objects_list[$i]##*/} downloaded from bucket" + + echo "Comparing files: ${objects_list[$i]} and ${objects_list[$i]}_s3_object${i}" + cmp "${objects_list[$i]}" "${objects_list[$i]}_s3_object${i}" + check_result $? "The files are not equal" "The files are equal" + + s3cmd del "${bucket}/${objects_list[$i]##*/}" $params + check_result $? "Error during s3cmd execution" "File from bucket is deleted" + done + + s3cmd del --recursive --force $bucket $params + check_result $? "Error during s3cmd execution" "Bucket is deleted" + + else + echo "Error during s3cmd execution" + exit 1 + fi +} + +function check_result () +{ + red='\033[0;31m' + green='\033[0;32m' + bw='\033[0m' + if [ "$1" -ne 0 ]; then + echo -e "${red}$2${bw}" + exit 1 + else + echo -e "${green}$3${bw}" + fi +} + +function prepare_objects () +{ + echo "Preparing ${total_objects} files for test" + for i in $(seq $total_objects); do + objects_list[$i]="$(mktemp -p "$tmpdir")" + echo "${objects_list[$i]}" + dd if=/dev/urandom of="${objects_list[$i]}" bs=1M count=8 + done +} + +prepare_objects + +if [ "$RGW_TEST_TYPE" == RGW_KS ]; +then + echo "--> Keystone is enabled. Calling function to test keystone based auth " + rgw_keystone_bucket_validation +fi + +if [ "$RGW_TEST_TYPE" == RGW_S3 ]; +then + echo "--> S3 is enabled. Calling function to test S3 based auth " + rgw_s3_bucket_validation +fi diff --git a/ceph-rgw/templates/bin/_init-dirs.sh.tpl b/ceph-rgw/templates/bin/_init-dirs.sh.tpl new file mode 100644 index 0000000000..9ab21097cc --- /dev/null +++ b/ceph-rgw/templates/bin/_init-dirs.sh.tpl @@ -0,0 +1,42 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +export LC_ALL=C +: "${HOSTNAME:=$(uname -n)}" +: "${RGW_NAME:=${HOSTNAME}}" +: "${RGW_BOOTSTRAP_KEYRING:=/var/lib/ceph/bootstrap-rgw/${CLUSTER}.keyring}" + +for keyring in ${RGW_BOOTSTRAP_KEYRING}; do + mkdir -p "$(dirname "$keyring")" +done + +# Let's create the ceph directories +for DIRECTORY in radosgw tmp; do + mkdir -p "/var/lib/ceph/${DIRECTORY}" +done + +# Create socket directory +mkdir -p /run/ceph + +# Creating rados directories +mkdir -p "/var/lib/ceph/radosgw/${RGW_NAME}" + +# Clean the folder +rm -f "$(dirname "${RGW_BOOTSTRAP_KEYRING}"/*)" + +# Adjust the owner of all those directories +chown -R ceph. /run/ceph/ /var/lib/ceph/* diff --git a/ceph-rgw/templates/bin/_rgw-restart.sh.tpl b/ceph-rgw/templates/bin/_rgw-restart.sh.tpl new file mode 100644 index 0000000000..a89645b46f --- /dev/null +++ b/ceph-rgw/templates/bin/_rgw-restart.sh.tpl @@ -0,0 +1,25 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +export LC_ALL=C +TIMEOUT="{{ .Values.conf.rgw_restart.timeout | default 600 }}s" + +kubectl rollout restart deployment ceph-rgw +kubectl rollout status --timeout=${TIMEOUT} deployment ceph-rgw + +if [ "$?" -ne 0 ]; then + echo "Ceph rgw deployment was not able to restart in ${TIMEOUT}" +fi diff --git a/ceph-rgw/templates/bin/rgw/_init.sh.tpl b/ceph-rgw/templates/bin/rgw/_init.sh.tpl new file mode 100644 index 0000000000..1e52bdcde9 --- /dev/null +++ b/ceph-rgw/templates/bin/rgw/_init.sh.tpl @@ -0,0 +1,86 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +export LC_ALL=C + +: "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}" +: "${EP:=ceph-mon-discovery}" +{{- if empty .Values.endpoints.ceph_mon.namespace -}} +MON_NS=ceph +{{ else }} +MON_NS={{ .Values.endpoints.ceph_mon.namespace }} +{{- end }} + +{{ include "helm-toolkit.snippets.mon_host_from_k8s_ep" . }} + +if [[ ! -e ${CEPH_CONF}.template ]]; then + echo "ERROR- ${CEPH_CONF}.template must exist." + exit 1 +fi + +ENDPOINT=$(mon_host_from_k8s_ep "${MON_NS}" "${EP}") + +if [[ -z "${ENDPOINT}" ]]; then + /bin/sh -c -e "cat ${CEPH_CONF}.template | tee ${CEPH_CONF}" || true +else + /bin/sh -c -e "cat ${CEPH_CONF}.template | sed 's#mon_host.*#mon_host = ${ENDPOINT}#g' | tee ${CEPH_CONF}" || true +fi + +cat >> ${CEPH_CONF} < /tmp/ceph-rbd-pool.json + kubectl -n ${CEPH_NS} delete job ceph-rbd-pool + jq 'del(.spec.selector) | + del(.spec.template.metadata.creationTimestamp) | + del(.spec.template.metadata.labels) | + del(.metadata.creationTimestamp) | + del(.metadata.uid) | + del(.status)' /tmp/ceph-rbd-pool.json | \ + kubectl create -f - + + while [[ -z "$(kubectl -n ${CEPH_NS} get pods | grep ceph-rbd-pool | grep Completed)" ]] + do + sleep 5 + done +fi diff --git a/ceph-rgw/templates/bin/rgw/_start.sh.tpl b/ceph-rgw/templates/bin/rgw/_start.sh.tpl new file mode 100644 index 0000000000..477fe91ae4 --- /dev/null +++ b/ceph-rgw/templates/bin/rgw/_start.sh.tpl @@ -0,0 +1,64 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +export LC_ALL=C +: "${CEPH_GET_ADMIN_KEY:=0}" +: "${RGW_NAME:=$(uname -n)}" +: "${RGW_ZONEGROUP:=}" +: "${RGW_ZONE:=}" +: "${ADMIN_KEYRING:=/etc/ceph/${CLUSTER}.client.admin.keyring}" +: "${RGW_KEYRING:=/var/lib/ceph/radosgw/${RGW_NAME}/keyring}" +: "${RGW_BOOTSTRAP_KEYRING:=/var/lib/ceph/bootstrap-rgw/${CLUSTER}.keyring}" + +if [[ ! -e "/etc/ceph/${CLUSTER}.conf" ]]; then + echo "ERROR- /etc/ceph/${CLUSTER}.conf must exist; get it from your existing mon" + exit 1 +fi + +if [ "${CEPH_GET_ADMIN_KEY}" -eq 1 ]; then + if [[ ! -e "${ADMIN_KEYRING}" ]]; then + echo "ERROR- ${ADMIN_KEYRING} must exist; get it from your existing mon" + exit 1 + fi +fi + +# Check to see if our RGW has been initialized +if [ ! -e "${RGW_KEYRING}" ]; then + + if [ ! -e "${RGW_BOOTSTRAP_KEYRING}" ]; then + echo "ERROR- ${RGW_BOOTSTRAP_KEYRING} must exist. You can extract it from your current monitor by running 'ceph auth get client.bootstrap-rgw -o ${RGW_BOOTSTRAP_KEYRING}'" + exit 1 + fi + + timeout 10 ceph --cluster "${CLUSTER}" --name "client.bootstrap-rgw" --keyring "${RGW_BOOTSTRAP_KEYRING}" health || exit 1 + + # Generate the RGW key + ceph --cluster "${CLUSTER}" --name "client.bootstrap-rgw" --keyring "${RGW_BOOTSTRAP_KEYRING}" auth get-or-create "client.rgw.${RGW_NAME}" osd 'allow rwx' mon 'allow rw' -o "${RGW_KEYRING}" + chown ceph. "${RGW_KEYRING}" + chmod 0600 "${RGW_KEYRING}" +fi + +/usr/bin/radosgw \ + --cluster "${CLUSTER}" \ + --setuser "ceph" \ + --setgroup "ceph" \ + -d \ + -n "client.rgw.${RGW_NAME}" \ + -k "${RGW_KEYRING}" \ + --rgw-socket-path="" \ + --rgw-zonegroup="${RGW_ZONEGROUP}" \ + --rgw-zone="${RGW_ZONE}" diff --git a/ceph-rgw/templates/bin/utils/_checkDNS.sh.tpl b/ceph-rgw/templates/bin/utils/_checkDNS.sh.tpl new file mode 100644 index 0000000000..b7e360b2fe --- /dev/null +++ b/ceph-rgw/templates/bin/utils/_checkDNS.sh.tpl @@ -0,0 +1,38 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +: "${CEPH_CONF:="/etc/ceph/${CLUSTER}.conf"}" +ENDPOINT="{$1}" + +function check_mon_dns () { + GREP_CMD=$(grep -rl 'ceph-mon' ${CEPH_CONF}) + + if [[ "${ENDPOINT}" == "{up}" ]]; then + echo "If DNS is working, we are good here" + elif [[ "${ENDPOINT}" != "" ]]; then + if [[ ${GREP_CMD} != "" ]]; then + # No DNS, write CEPH MONs IPs into ${CEPH_CONF} + sh -c -e "cat ${CEPH_CONF}.template | sed 's/mon_host.*/mon_host = ${ENDPOINT}/g' | tee ${CEPH_CONF}" > /dev/null 2>&1 + else + echo "endpoints are already cached in ${CEPH_CONF}" + exit + fi + fi +} + +check_mon_dns + +exit diff --git a/ceph-rgw/templates/certificates.yaml b/ceph-rgw/templates/certificates.yaml new file mode 100644 index 0000000000..06aca44758 --- /dev/null +++ b/ceph-rgw/templates/certificates.yaml @@ -0,0 +1,20 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{ $object_store_name := "object_store" }} +{{- if .Values.conf.rgw_s3.enabled }} +{{ $object_store_name = "ceph_object_store" }} +{{- end }} +{{- if .Values.manifests.certificates }} +{{ dict "envAll" . "service" $object_store_name "type" "internal" | include "helm-toolkit.manifests.certificates" }} +{{- end }} diff --git a/ceph-rgw/templates/configmap-bin-ks.yaml b/ceph-rgw/templates/configmap-bin-ks.yaml new file mode 100644 index 0000000000..5fca7e263a --- /dev/null +++ b/ceph-rgw/templates/configmap-bin-ks.yaml @@ -0,0 +1,29 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.configmap_bin_ks .Values.conf.rgw_ks.enabled }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ceph-rgw-bin-ks +data: + ks-service.sh: | +{{- include "helm-toolkit.scripts.keystone_service" . | indent 4 }} + ks-endpoints.sh: | +{{- include "helm-toolkit.scripts.keystone_endpoints" . | indent 4 }} + ks-user.sh: | +{{- include "helm-toolkit.scripts.keystone_user" . | indent 4 }} +{{- end }} diff --git a/ceph-rgw/templates/configmap-bin.yaml b/ceph-rgw/templates/configmap-bin.yaml new file mode 100644 index 0000000000..aa970d4103 --- /dev/null +++ b/ceph-rgw/templates/configmap-bin.yaml @@ -0,0 +1,55 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.configmap_bin .Values.deployment.ceph }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ceph-rgw-bin +data: +{{- if .Values.images.local_registry.active }} + image-repo-sync.sh: | +{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} +{{- end }} + +{{- if .Values.bootstrap.enabled }} + bootstrap.sh: | +{{ tuple "bin/_bootstrap.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} + rgw-restart.sh: | +{{ tuple "bin/_rgw-restart.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + init-dirs.sh: | +{{ tuple "bin/_init-dirs.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + + rgw-start.sh: | +{{ tuple "bin/rgw/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + rgw-init.sh: | +{{ tuple "bin/rgw/_init.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + rerun-pool-job.sh: | +{{ tuple "bin/rgw/_rerun-pool-job.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + storage-init.sh: | +{{ tuple "bin/_ceph-rgw-storage-init.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + ceph-admin-keyring.sh: | +{{ tuple "bin/_ceph-admin-keyring.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + rgw-s3-admin.sh: | +{{- include "helm-toolkit.scripts.create_s3_user" . | indent 4 }} + create-rgw-placement-targets.sh: | +{{ tuple "bin/_create-rgw-placement-targets.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + helm-tests.sh: | +{{ tuple "bin/_helm-tests.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + utils-checkDNS.sh: | +{{ tuple "bin/utils/_checkDNS.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} diff --git a/ceph-rgw/templates/configmap-ceph-rgw-templates.yaml b/ceph-rgw/templates/configmap-ceph-rgw-templates.yaml new file mode 100644 index 0000000000..cf0012762e --- /dev/null +++ b/ceph-rgw/templates/configmap-ceph-rgw-templates.yaml @@ -0,0 +1,25 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.configmap_ceph_templates .Values.manifests.job_ceph_rgw_storage_init }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ printf "%s-%s" $envAll.Release.Name "ceph-templates" | quote }} +data: + bootstrap.keyring.rgw: | +{{ .Values.conf.templates.keyring.bootstrap.rgw | indent 4 }} +{{- end }} diff --git a/ceph-rgw/templates/configmap-etc-client.yaml b/ceph-rgw/templates/configmap-etc-client.yaml new file mode 100644 index 0000000000..2e7febbf73 --- /dev/null +++ b/ceph-rgw/templates/configmap-etc-client.yaml @@ -0,0 +1,54 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "ceph.configmap.etc" }} +{{- $configMapName := index . 0 }} +{{- $envAll := index . 1 }} +{{- with $envAll }} + +{{- if or (.Values.deployment.ceph) (.Values.deployment.client_secrets) }} + +{{- if empty .Values.conf.ceph.global.mon_host -}} +{{- $monHost := tuple "ceph_mon" "internal" "mon_msgr2" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} +{{- $_ := $monHost | set .Values.conf.ceph.global "mon_host" -}} +{{- end -}} + + +{{- if empty .Values.conf.ceph.osd.cluster_network -}} +{{- $_ := .Values.network.cluster | set .Values.conf.ceph.osd "cluster_network" -}} +{{- end -}} + +{{- if empty .Values.conf.ceph.osd.public_network -}} +{{- $_ := .Values.network.public | set .Values.conf.ceph.osd "public_network" -}} +{{- end -}} + +{{- if empty .Values.conf.rgw_ks.config.rgw_swift_url -}} +{{- $_ := tuple "object_store" "public" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | trimSuffix .Values.endpoints.object_store.path.default | set .Values.conf.rgw_ks.config "rgw_swift_url" -}} +{{- end -}} + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $configMapName }} +data: + ceph.conf: | +{{ include "helm-toolkit.utils.to_ini" .Values.conf.ceph | indent 4 }} + +{{- end }} +{{- end }} +{{- end }} +{{- if .Values.manifests.configmap_etc }} +{{- list "ceph-rgw-etc" . | include "ceph.configmap.etc" }} +{{- end }} diff --git a/ceph-rgw/templates/deployment-rgw.yaml b/ceph-rgw/templates/deployment-rgw.yaml new file mode 100644 index 0000000000..1fde8afe57 --- /dev/null +++ b/ceph-rgw/templates/deployment-rgw.yaml @@ -0,0 +1,274 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "readinessProbeTemplate" }} +{{- $object_store_name := "object_store" }} +{{- if .Values.conf.rgw_s3.enabled }} +{{ $object_store_name = "ceph_object_store" }} +{{- end }} +httpGet: + path: / + port: {{ tuple $object_store_name "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + scheme: {{ tuple $object_store_name "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" | upper }} +{{- end }} + +{{- define "livenessProbeTemplate" }} +{{- $object_store_name := "object_store" }} +{{- if .Values.conf.rgw_s3.enabled }} +{{ $object_store_name = "ceph_object_store" }} +{{- end }} +httpGet: + path: / + port: {{ tuple $object_store_name "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + scheme: {{ tuple $object_store_name "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" | upper }} +{{- end }} + +{{- if and .Values.manifests.deployment_rgw ( and .Values.deployment.ceph .Values.conf.features.rgw ) }} +{{- $envAll := . }} + +{{ $object_store_name := "object_store" }} +{{ $tls_secret := .Values.secrets.tls.object_store.api.internal | quote }} +{{- if .Values.conf.rgw_s3.enabled }} +{{ $object_store_name = "ceph_object_store" }} +{{ $tls_secret = .Values.secrets.tls.ceph_object_store.api.internal | quote }} +{{- end }} + +{{- $serviceAccountName := "ceph-rgw" }} +{{- $checkDnsServiceAccountName := "ceph-checkdns" }} + +{{- $_ := set $envAll.Values "__depParams" ( list ) }} +{{- if .Values.conf.rgw_ks.enabled -}} +{{- $__updateDepParams := append $envAll.Values.__depParams "keystone" -}} +{{- $_ := set $envAll.Values "__depParams" $__updateDepParams -}} +{{- end -}} +{{- if .Values.conf.rgw_s3.enabled -}} +{{- $__updateDepParams := append $envAll.Values.__depParams "s3" -}} +{{- $_ := set $envAll.Values "__depParams" $__updateDepParams -}} +{{- end -}} +{{- $dependencyOpts := dict "envAll" $envAll "dependencyMixinParam" $envAll.Values.__depParams "dependencyKey" "rgw" -}} +{{- $_ := include "helm-toolkit.utils.dependency_resolver" $dependencyOpts | toString | fromYaml }} +{{ tuple $envAll "pod_dependency" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ printf "%s-%s" $serviceAccountName $envAll.Release.Namespace }} + namespace: {{ .Values.endpoints.ceph_mon.namespace }} +rules: + - apiGroups: + - "" + resources: + - endpoints + verbs: + - get + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ printf "%s-%s" $serviceAccountName $envAll.Release.Namespace }} + namespace: {{ .Values.endpoints.ceph_mon.namespace }} +roleRef: + kind: Role + name: {{ printf "%s-%s" $serviceAccountName $envAll.Release.Namespace }} + apiGroup: rbac.authorization.k8s.io +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +--- +# This role bindig refers to the ClusterRole for +# check-dns deployment. +# See: openstack-helm-infra/ceph-client/deployment-checkdns.yaml +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ printf "%s-from-%s-to-%s" $checkDnsServiceAccountName $envAll.Values.endpoints.ceph_mon.namespace $envAll.Release.Namespace }} + namespace: {{ $envAll.Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: clusterrole-checkdns +subjects: + - kind: ServiceAccount + name: {{ $checkDnsServiceAccountName }} + namespace: {{ .Values.endpoints.ceph_mon.namespace }} +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: ceph-rgw + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "ceph" "rgw" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + replicas: {{ .Values.pod.replicas.rgw }} + selector: + matchLabels: +{{ tuple $envAll "ceph" "rgw" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} +{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "ceph" "rgw" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-etc-client-hash: {{ tuple "configmap-etc-client.yaml" . | include "helm-toolkit.utils.hash" }} + secret-keystone-rgw-hash: {{ tuple "secret-keystone-rgw.yaml" . | include "helm-toolkit.utils.hash" }} +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} +{{ dict "envAll" $envAll "podName" "ceph-rgw" "containerNames" (list "init" "ceph-rgw" "ceph-init-dirs" "ceph-rgw-init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "rgw" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + affinity: +{{ tuple $envAll "ceph" "rgw" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} +{{ tuple $envAll "rgw" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }} + nodeSelector: + {{ .Values.labels.rgw.node_selector_key }}: {{ .Values.labels.rgw.node_selector_value }} + initContainers: +{{ tuple $envAll "pod_dependency" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + - name: ceph-init-dirs +{{ tuple $envAll "ceph_rgw" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ dict "envAll" $envAll "application" "rgw" "container" "init_dirs" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/init-dirs.sh + env: + - name: CLUSTER + value: "ceph" + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: pod-run + mountPath: /run + - name: pod-etc-ceph + mountPath: /etc/ceph + - name: ceph-rgw-bin + mountPath: /tmp/init-dirs.sh + subPath: init-dirs.sh + readOnly: true + - name: pod-var-lib-ceph + mountPath: /var/lib/ceph + readOnly: false + - name: ceph-rgw-init +{{ tuple $envAll "ceph_rgw" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.rgw | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "rgw" "container" "rgw_init" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + env: + - name: CLUSTER + value: "ceph" + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name +{{ if .Values.conf.rgw_ks.enabled }} +{{- with $env := dict "ksUserSecret" .Values.secrets.identity.user_rgw "useCA" .Values.manifests.certificates }} +{{- include "helm-toolkit.snippets.keystone_openrc_env_vars" $env | indent 12 }} +{{- end }} + - name: KEYSTONE_URL + value: {{ tuple "identity" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | trimSuffix .Values.endpoints.identity.path.default | quote }} +{{ end }} + - name: RGW_FRONTEND_PORT + value: "{{ tuple $object_store_name "internal" "api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}" + command: + - /tmp/rgw-init.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: pod-run + mountPath: /run + - name: pod-etc-ceph + mountPath: /etc/ceph + - name: ceph-rgw-bin + mountPath: /tmp/rgw-init.sh + subPath: rgw-init.sh + readOnly: true + - name: ceph-rgw-etc + mountPath: /etc/ceph/ceph.conf.template + subPath: ceph.conf + readOnly: true +{{- if .Values.conf.rgw_ks.enabled }} +{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.object_store.api.keystone | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }} +{{- end }} + containers: + - name: ceph-rgw +{{ tuple $envAll "ceph_rgw" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.rgw | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "rgw" "container" "rgw" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + env: + - name: CLUSTER + value: "ceph" + - name: RGW_FRONTEND_PORT + value: "{{ tuple $object_store_name "internal" "api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}" + command: + - /tmp/rgw-start.sh + ports: + - containerPort: {{ tuple $object_store_name "internal" "api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +{{ dict "envAll" . "component" "api" "container" "ceph-rgw" "type" "liveness" "probeTemplate" (include "livenessProbeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | trim | indent 10 }} +{{ dict "envAll" . "component" "api" "container" "ceph-rgw" "type" "readiness" "probeTemplate" (include "readinessProbeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | trim | indent 10 }} + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: pod-run + mountPath: /run + - name: pod-etc-ceph + mountPath: /etc/ceph + - name: ceph-rgw-bin + mountPath: /tmp/rgw-start.sh + subPath: rgw-start.sh + readOnly: true + - name: ceph-rgw-bin + mountPath: /tmp/utils-checkDNS.sh + subPath: utils-checkDNS.sh + readOnly: true + - name: ceph-rgw-etc + mountPath: /etc/ceph/ceph.conf.template + subPath: ceph.conf + readOnly: true + - name: ceph-bootstrap-rgw-keyring + mountPath: /var/lib/ceph/bootstrap-rgw/ceph.keyring + subPath: ceph.keyring + readOnly: false + - name: pod-var-lib-ceph + mountPath: /var/lib/ceph + readOnly: false +{{- dict "enabled" .Values.manifests.certificates "name" $tls_secret "path" "/etc/tls" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: pod-run + emptyDir: + medium: "Memory" + - name: pod-etc-ceph + emptyDir: {} + - name: ceph-rgw-bin + configMap: + name: ceph-rgw-bin + defaultMode: 0555 + - name: ceph-rgw-etc + configMap: + name: ceph-rgw-etc + defaultMode: 0444 + - name: pod-var-lib-ceph + emptyDir: {} + - name: ceph-bootstrap-rgw-keyring + secret: + secretName: {{ .Values.secrets.keyrings.rgw }} +{{- dict "enabled" .Values.manifests.certificates "name" $tls_secret | include "helm-toolkit.snippets.tls_volume" | indent 8 }} +{{- if .Values.conf.rgw_ks.enabled }} +{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.object_store.api.keystone | include "helm-toolkit.snippets.tls_volume" | indent 8 }} +{{- end }} +{{- end }} diff --git a/ceph-rgw/templates/ingress-rgw.yaml b/ceph-rgw/templates/ingress-rgw.yaml new file mode 100644 index 0000000000..bddef84806 --- /dev/null +++ b/ceph-rgw/templates/ingress-rgw.yaml @@ -0,0 +1,26 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.ingress_rgw ( and .Values.deployment.ceph (and .Values.network.api.ingress.public .Values.conf.features.rgw) ) }} +{{- $ingressOpts := dict "envAll" . "backendServiceType" "object_store" "backendPort" "ceph-rgw" -}} +{{- if .Values.manifests.certificates }} +{{- if .Values.conf.rgw_ks.enabled }} +{{- $ingressOpts = dict "envAll" . "backendServiceType" "object_store" "backendPort" "ceph-rgw" "certIssuer" .Values.endpoints.object_store.host_fqdn_override.default.tls.issuerRef.name -}} +{{- end }} +{{- if .Values.conf.rgw_s3.enabled }} +{{- $ingressOpts = dict "envAll" . "backendServiceType" "ceph_object_store" "backendPort" "ceph-rgw" "certIssuer" .Values.endpoints.ceph_object_store.host_fqdn_override.default.tls.issuerRef.name -}} +{{- end }} +{{- end }} +{{ $ingressOpts | include "helm-toolkit.manifests.ingress" }} +{{- end }} diff --git a/ceph-rgw/templates/job-bootstrap.yaml b/ceph-rgw/templates/job-bootstrap.yaml new file mode 100644 index 0000000000..6368969133 --- /dev/null +++ b/ceph-rgw/templates/job-bootstrap.yaml @@ -0,0 +1,131 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_bootstrap .Values.bootstrap.enabled }} +{{- $envAll := . }} + +{{- $serviceAccountName := "ceph-rgw-bootstrap" }} +{{ tuple $envAll "bootstrap" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $serviceAccountName }} +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create + - update + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $serviceAccountName }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: ceph-rgw-bootstrap + labels: +{{ tuple $envAll "ceph" "bootstrap" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +spec: + template: + metadata: + labels: +{{ tuple $envAll "ceph" "bootstrap" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} +{{ dict "envAll" $envAll "podName" "ceph-rgw-bootstrap" "containerNames" (list "ceph-keyring-placement" "init" "ceph-rgw-bootstrap") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "bootstrap" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} + initContainers: +{{ tuple $envAll "bootstrap" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + - name: ceph-keyring-placement +{{ tuple $envAll "ceph_config_helper" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ dict "envAll" $envAll "application" "bootstrap" "container" "keyring_placement" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/ceph-admin-keyring.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: pod-etc-ceph + mountPath: /etc/ceph + - name: ceph-rgw-bin + mountPath: /tmp/ceph-admin-keyring.sh + subPath: ceph-admin-keyring.sh + readOnly: true + - name: ceph-rgw-admin-keyring + mountPath: /tmp/client-keyring + subPath: key + readOnly: true + containers: + - name: ceph-rgw-bootstrap +{{ tuple $envAll "ceph_bootstrap" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.bootstrap | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "bootstrap" "container" "bootstrap" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/bootstrap.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: pod-etc-ceph + mountPath: /etc/ceph + - name: ceph-rgw-bin + mountPath: /tmp/bootstrap.sh + subPath: bootstrap.sh + readOnly: true + - name: ceph-rgw-etc + mountPath: /etc/ceph/ceph.conf + subPath: ceph.conf + readOnly: true + - name: ceph-rgw-admin-keyring + mountPath: /tmp/client-keyring + subPath: key + readOnly: true + volumes: + - name: pod-tmp + emptyDir: {} + - name: pod-etc-ceph + emptyDir: {} + - name: ceph-rgw-bin + configMap: + name: ceph-rgw-bin + defaultMode: 0555 + - name: ceph-rgw-etc + configMap: + name: {{ .Values.ceph_client.configmap }} + defaultMode: 0444 + - name: ceph-rgw-admin-keyring + secret: + secretName: {{ .Values.secrets.keyrings.admin | quote }} +{{- end }} diff --git a/ceph-rgw/templates/job-image-repo-sync.yaml b/ceph-rgw/templates/job-image-repo-sync.yaml new file mode 100644 index 0000000000..8739079761 --- /dev/null +++ b/ceph-rgw/templates/job-image-repo-sync.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "ceph-rgw" -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/ceph-rgw/templates/job-ks-endpoints.yaml b/ceph-rgw/templates/job-ks-endpoints.yaml new file mode 100644 index 0000000000..c60be015bf --- /dev/null +++ b/ceph-rgw/templates/job-ks-endpoints.yaml @@ -0,0 +1,21 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_ks_endpoints .Values.conf.rgw_ks.enabled }} +{{- $ksServiceJob := dict "envAll" . "configMapBin" "ceph-rgw-bin-ks" "serviceName" "ceph" "serviceTypes" ( tuple "object-store" ) -}} +{{- if .Values.manifests.certificates -}} +{{- $_ := set $ksServiceJob "tlsSecret" .Values.secrets.tls.object_store.api.internal -}} +{{- end -}} +{{ $ksServiceJob | include "helm-toolkit.manifests.job_ks_endpoints" }} +{{- end }} diff --git a/ceph-rgw/templates/job-ks-service.yaml b/ceph-rgw/templates/job-ks-service.yaml new file mode 100644 index 0000000000..f62040a6ba --- /dev/null +++ b/ceph-rgw/templates/job-ks-service.yaml @@ -0,0 +1,21 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_ks_service .Values.conf.rgw_ks.enabled }} +{{- $ksServiceJob := dict "envAll" . "configMapBin" "ceph-rgw-bin-ks" "serviceName" "ceph" "serviceTypes" ( tuple "object-store" ) -}} +{{- if .Values.manifests.certificates -}} +{{- $_ := set $ksServiceJob "tlsSecret" .Values.secrets.tls.object_store.api.internal -}} +{{- end -}} +{{ $ksServiceJob | include "helm-toolkit.manifests.job_ks_service" }} +{{- end }} diff --git a/ceph-rgw/templates/job-ks-user.yaml b/ceph-rgw/templates/job-ks-user.yaml new file mode 100644 index 0000000000..8f6e12a5c4 --- /dev/null +++ b/ceph-rgw/templates/job-ks-user.yaml @@ -0,0 +1,21 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_ks_user .Values.conf.rgw_ks.enabled }} +{{- $ksUserJob := dict "envAll" . "configMapBin" "ceph-rgw-bin-ks" "serviceName" "ceph" "serviceUser" "swift" -}} +{{- if .Values.manifests.certificates -}} +{{- $_ := set $ksUserJob "tlsSecret" .Values.secrets.tls.object_store.api.internal -}} +{{- end -}} +{{ $ksUserJob | include "helm-toolkit.manifests.job_ks_user" }} +{{- end }} diff --git a/ceph-rgw/templates/job-rgw-placement-targets.yaml b/ceph-rgw/templates/job-rgw-placement-targets.yaml new file mode 100644 index 0000000000..45b9486adc --- /dev/null +++ b/ceph-rgw/templates/job-rgw-placement-targets.yaml @@ -0,0 +1,133 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_rgw_placement_targets .Values.conf.features.rgw }} +{{- $envAll := . }} + +{{- $serviceAccountName := "rgw-placement-targets" }} +{{ tuple $envAll "rgw_placement_targets" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $serviceAccountName }} +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create + - update + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $serviceAccountName }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: ceph-rgw-placement-targets + labels: +{{ tuple $envAll "ceph" "rgw-placement-targets" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +spec: + template: + metadata: + labels: +{{ tuple $envAll "ceph" "rgw-placement-targets" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} +{{ dict "envAll" $envAll "podName" "ceph-rgw-placement-targets" "containerNames" (list "ceph-keyring-placement" "init" "create-rgw-placement-targets") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "rgw_placement_targets" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} + initContainers: +{{ tuple $envAll "rgw_placement_targets" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + - name: ceph-keyring-placement +{{ tuple $envAll "ceph_config_helper" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ dict "envAll" $envAll "application" "rgw_placement_targets" "container" "keyring_placement" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/ceph-admin-keyring.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: pod-etc-ceph + mountPath: /etc/ceph + - name: ceph-rgw-bin + mountPath: /tmp/ceph-admin-keyring.sh + subPath: ceph-admin-keyring.sh + readOnly: true + - name: ceph-keyring + mountPath: /tmp/client-keyring + subPath: key + readOnly: true + containers: + - name: create-rgw-placement-targets + image: {{ .Values.images.tags.rgw_placement_targets }} + imagePullPolicy: {{ .Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.rgw_placement_targets | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "rgw_placement_targets" "container" "create_rgw_placement_targets" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/create-rgw-placement-targets.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: pod-etc-ceph + mountPath: /etc/ceph + - name: ceph-rgw-bin + mountPath: /tmp/create-rgw-placement-targets.sh + subPath: create-rgw-placement-targets.sh + readOnly: true + - name: ceph-rgw-etc + mountPath: /etc/ceph/ceph.conf + subPath: ceph.conf + readOnly: true + - name: ceph-keyring + mountPath: /tmp/client-keyring + subPath: key + readOnly: true + volumes: + - name: pod-tmp + emptyDir: {} + - name: pod-etc-ceph + emptyDir: {} + - name: ceph-rgw-bin + configMap: + name: ceph-rgw-bin + defaultMode: 0555 + - name: ceph-rgw-etc + configMap: + name: ceph-rgw-etc + defaultMode: 0444 + - name: ceph-keyring + secret: + secretName: {{ .Values.secrets.keyrings.admin | quote }} +{{- end }} diff --git a/ceph-rgw/templates/job-rgw-pool.yaml b/ceph-rgw/templates/job-rgw-pool.yaml new file mode 100644 index 0000000000..dfe9c8f00b --- /dev/null +++ b/ceph-rgw/templates/job-rgw-pool.yaml @@ -0,0 +1,115 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# This job is required for Reef and later because Ceph now disallows the +# creation of internal pools (pools names beginning with a ".") and the +# ceph-rbd-pool job therefore can't configure them if they don't yet exist. +# This job simply deletes and re-creates the ceph-rbd-pool job after deploying +# ceph-rgw so it can apply the correct configuration to the .rgw.root pool. + +{{- if and .Values.manifests.job_rgw_pool .Values.deployment.ceph }} +{{- $envAll := . }} + +{{- $serviceAccountName := "ceph-rgw-pool" }} +{{ tuple $envAll "rgw_pool" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ $serviceAccountName }}-{{ $envAll.Release.Namespace }} +rules: + - apiGroups: + - '' + resources: + - pods + - jobs + verbs: + - create + - get + - delete + - list + - apiGroups: + - 'batch' + resources: + - jobs + verbs: + - create + - get + - delete + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ $serviceAccountName }}-{{ $envAll.Release.Namespace }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ $serviceAccountName }}-{{ $envAll.Release.Namespace }} + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: ceph-rgw-pool + labels: +{{ tuple $envAll "ceph" "rbd-pool" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +spec: + template: + metadata: + name: ceph-rgw-pool + labels: +{{ tuple $envAll "ceph" "rbd-pool" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ dict "envAll" $envAll "podName" "ceph-rgw-pool" "containerNames" (list "ceph-rgw-pool" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "rgw_pool" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: {{ $envAll.Values.jobs.rgw_pool.restartPolicy | quote }} + affinity: +{{ tuple $envAll "ceph" "rbd-pool" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} + nodeSelector: + {{ $envAll.Values.labels.job.node_selector_key }}: {{ $envAll.Values.labels.job.node_selector_value }} + initContainers: +{{ tuple $envAll "rgw_pool" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: ceph-rgw-pool +{{ tuple $envAll "ceph_rgw_pool" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.rgw_pool | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "rgw_pool" "container" "rgw_pool" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/rerun-pool-job.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: ceph-rgw-bin + mountPath: /tmp/rerun-pool-job.sh + subPath: rerun-pool-job.sh + readOnly: true + volumes: + - name: pod-tmp + emptyDir: {} + - name: ceph-rgw-bin + configMap: + name: ceph-rgw-bin + defaultMode: 0555 + - name: pod-run + emptyDir: + medium: "Memory" +{{- end }} diff --git a/ceph-rgw/templates/job-rgw-restart.yaml b/ceph-rgw/templates/job-rgw-restart.yaml new file mode 100644 index 0000000000..fdbec8f9d7 --- /dev/null +++ b/ceph-rgw/templates/job-rgw-restart.yaml @@ -0,0 +1,91 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_rgw_restart }} +{{- $envAll := . }} + +{{- $serviceAccountName := printf "%s-%s" .Release.Name "rgw-restart" }} +{{ tuple $envAll "rgw_restart" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ $serviceAccountName }} +rules: + - apiGroups: + - 'apps' + resources: + - deployments + verbs: + - get + - list + - update + - patch + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ $serviceAccountName }} + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: ceph-rgw-restart + labels: +{{ tuple $envAll "ceph" "rgw-restart" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +spec: + template: + metadata: + labels: +{{ tuple $envAll "ceph" "rgw-restart" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} +{{ dict "envAll" $envAll "podName" "ceph-rgw-restart" "containerNames" (list "init" "ceph-rgw-restart") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "rgw_restart" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} + initContainers: +{{ tuple $envAll "rgw_restart" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: ceph-rgw-restart +{{ tuple $envAll "ceph_config_helper" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.rgw_restart | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "rgw_restart" "container" "ceph-rgw-restart" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/rgw-restart.sh + volumeMounts: + - name: ceph-rgw-bin + mountPath: /tmp/rgw-restart.sh + subPath: rgw-restart.sh + readOnly: true + volumes: + - name: ceph-rgw-bin + configMap: + name: ceph-rgw-bin + defaultMode: 0555 +{{- end }} diff --git a/ceph-rgw/templates/job-rgw-storage-init.yaml b/ceph-rgw/templates/job-rgw-storage-init.yaml new file mode 100644 index 0000000000..4c3a6ed3ea --- /dev/null +++ b/ceph-rgw/templates/job-rgw-storage-init.yaml @@ -0,0 +1,143 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_ceph_rgw_storage_init .Values.deployment.ceph }} +{{- $envAll := . }} + +{{- $serviceAccountName := "ceph-rgw-storage-init" }} +{{ tuple $envAll "rgw_storage_init" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $serviceAccountName }} +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create + - update + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $serviceAccountName }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: ceph-rgw-storage-init + labels: +{{ tuple $envAll "ceph" "rgw-storage-init" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + template: + metadata: + labels: +{{ tuple $envAll "ceph" "rgw-storage-init" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} +{{ dict "envAll" $envAll "podName" "ceph-rgw-storage-init" "containerNames" (list "ceph-keyring-placement" "init" "ceph-rgw-storage-init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "rgw_storage_init" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} + initContainers: +{{ tuple $envAll "rgw_storage_init" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + - name: ceph-keyring-placement +{{ tuple $envAll "ceph_config_helper" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ dict "envAll" $envAll "application" "rgw_storage_init" "container" "keyring_placement" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/ceph-admin-keyring.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: pod-etc-ceph + mountPath: /etc/ceph + - name: ceph-rgw-bin + mountPath: /tmp/ceph-admin-keyring.sh + subPath: ceph-admin-keyring.sh + readOnly: true + - name: ceph-keyring + mountPath: /tmp/client-keyring + subPath: key + readOnly: true + containers: + - name: ceph-rgw-storage-init +{{ tuple $envAll "ceph_config_helper" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.rgw_storage_init | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "rgw_storage_init" "container" "rgw_storage_init" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: STORAGE_BACKEND + value: "ceph-rgw" + command: + - /tmp/storage-init.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: pod-etc-ceph + mountPath: /etc/ceph + - name: ceph-rgw-bin + mountPath: /tmp/storage-init.sh + subPath: storage-init.sh + readOnly: true + - name: ceph-templates + mountPath: /tmp/ceph-templates + readOnly: true + - name: ceph-etc + mountPath: /etc/ceph/ceph.conf + subPath: ceph.conf + readOnly: true + - name: ceph-keyring + mountPath: /tmp/client-keyring + subPath: key + readOnly: true + volumes: + - name: pod-tmp + emptyDir: {} + - name: pod-etc-ceph + emptyDir: {} + - name: ceph-rgw-bin + configMap: + name: ceph-rgw-bin + defaultMode: 0555 + - name: ceph-etc + configMap: + name: {{ .Values.ceph_client.configmap }} + defaultMode: 0444 + - name: ceph-templates + configMap: + name: {{ printf "%s-%s" $envAll.Release.Name "ceph-templates" | quote }} + defaultMode: 0444 + - name: ceph-keyring + secret: + secretName: {{ .Values.secrets.keyrings.admin | quote }} +{{- end }} diff --git a/ceph-rgw/templates/job-s3-admin.yaml b/ceph-rgw/templates/job-s3-admin.yaml new file mode 100644 index 0000000000..d796395b72 --- /dev/null +++ b/ceph-rgw/templates/job-s3-admin.yaml @@ -0,0 +1,150 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_s3_admin ( and .Values.conf.features.rgw .Values.conf.rgw_s3.enabled ) }} +{{- $envAll := . }} + +{{- $serviceAccountName := "rgw-s3-admin" }} +{{ tuple $envAll "rgw_s3_admin" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} + +{{- $s3AdminSecret := .Values.secrets.rgw_s3.admin }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $serviceAccountName }} +rules: + - apiGroups: + - "" + resources: + - secrets + verbs: + - get + - create + - update + - patch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $serviceAccountName }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: ceph-rgw-s3-admin + labels: +{{ tuple $envAll "ceph" "rgw-s3-admin" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +spec: + template: + metadata: + labels: +{{ tuple $envAll "ceph" "rgw-s3-admin" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} +{{ dict "envAll" $envAll "podName" "ceph-rgw-s3-admin" "containerNames" (list "ceph-keyring-placement" "init" "create-s3-admin") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "rgw_s3_admin" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} + initContainers: +{{ tuple $envAll "rgw_s3_admin" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + - name: ceph-keyring-placement +{{ tuple $envAll "ceph_config_helper" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ dict "envAll" $envAll "application" "rgw_s3_admin" "container" "keyring_placement" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/ceph-admin-keyring.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: pod-etc-ceph + mountPath: /etc/ceph + - name: ceph-rgw-bin + mountPath: /tmp/ceph-admin-keyring.sh + subPath: ceph-admin-keyring.sh + readOnly: true + - name: ceph-keyring + mountPath: /tmp/client-keyring + subPath: key + readOnly: true + containers: + - name: create-s3-admin + image: {{ .Values.images.tags.rgw_s3_admin }} + imagePullPolicy: {{ .Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.rgw_s3_admin | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "rgw_s3_admin" "container" "create_s3_admin" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + env: + - name: S3_USERNAME + valueFrom: + secretKeyRef: + name: {{ $s3AdminSecret }} + key: S3_ADMIN_USERNAME + - name: S3_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ $s3AdminSecret }} + key: S3_ADMIN_ACCESS_KEY + - name: S3_SECRET_KEY + valueFrom: + secretKeyRef: + name: {{ $s3AdminSecret }} + key: S3_ADMIN_SECRET_KEY + command: + - /tmp/rgw-s3-admin.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: pod-etc-ceph + mountPath: /etc/ceph + - name: ceph-rgw-bin + mountPath: /tmp/rgw-s3-admin.sh + subPath: rgw-s3-admin.sh + readOnly: true + - name: ceph-rgw-etc + mountPath: /etc/ceph/ceph.conf + subPath: ceph.conf + readOnly: true + - name: ceph-keyring + mountPath: /tmp/client-keyring + subPath: key + readOnly: true + volumes: + - name: pod-tmp + emptyDir: {} + - name: pod-etc-ceph + emptyDir: {} + - name: ceph-rgw-bin + configMap: + name: ceph-rgw-bin + defaultMode: 0555 + - name: ceph-rgw-etc + configMap: + name: ceph-rgw-etc + defaultMode: 0444 + - name: ceph-keyring + secret: + secretName: {{ .Values.secrets.keyrings.admin | quote }} +{{- end }} diff --git a/ceph-rgw/templates/network_policy.yaml b/ceph-rgw/templates/network_policy.yaml new file mode 100644 index 0000000000..bd5437f29c --- /dev/null +++ b/ceph-rgw/templates/network_policy.yaml @@ -0,0 +1,16 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- if .Values.manifests.network_policy -}} +{{- $netpol_opts := dict "envAll" . "key" "rgw" "labels" (dict "application" "ceph" "component" "rgw") -}} +{{ $netpol_opts | include "helm-toolkit.manifests.kubernetes_network_policy" }} +{{- end -}} diff --git a/ceph-rgw/templates/pod-helm-tests.yaml b/ceph-rgw/templates/pod-helm-tests.yaml new file mode 100644 index 0000000000..54a0f8706b --- /dev/null +++ b/ceph-rgw/templates/pod-helm-tests.yaml @@ -0,0 +1,126 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{- if and .Values.manifests.helm_tests .Values.deployment.ceph }} +{{- $envAll := . }} + +{{- $serviceAccountName := printf "%s-%s" $envAll.Release.Name "test" }} +{{ tuple $envAll "tests" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: v1 +kind: Pod +metadata: + name: {{ $serviceAccountName }} + labels: +{{ tuple $envAll "ceph" "rgw-test" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: + "helm.sh/hook": test-success +{{ dict "envAll" $envAll "podName" "ceph-rgw-test" "containerNames" (list "ceph-rgw-ks-validation" "ceph-rgw-s3-validation") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 4 }} +spec: +{{ dict "envAll" $envAll "application" "rgw_test" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 2 }} + restartPolicy: Never + serviceAccountName: {{ $serviceAccountName }} + nodeSelector: + {{ .Values.labels.test.node_selector_key }}: {{ .Values.labels.test.node_selector_value }} + containers: +{{ if .Values.conf.rgw_ks.enabled }} + - name: ceph-rgw-ks-validation +{{ tuple $envAll "ceph_config_helper" | include "helm-toolkit.snippets.image" | indent 6 }} +{{ tuple $envAll $envAll.Values.pod.resources.tests | include "helm-toolkit.snippets.kubernetes_resources" | indent 6 }} +{{ dict "envAll" $envAll "application" "rgw_test" "container" "ceph_rgw_ks_validation" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 6 }} + env: +{{- with $env := dict "ksUserSecret" .Values.secrets.identity.user_rgw "useCA" .Values.manifests.certificates }} +{{- include "helm-toolkit.snippets.keystone_openrc_env_vars" $env | indent 8 }} + - name: OS_AUTH_TYPE + valueFrom: + secretKeyRef: + name: {{ $.Values.secrets.identity.user_rgw }} + key: OS_AUTH_TYPE + - name: OS_TENANT_NAME + valueFrom: + secretKeyRef: + name: {{ $.Values.secrets.identity.user_rgw }} + key: OS_TENANT_NAME +{{- end }} + - name: "RGW_TEST_TYPE" + value: "RGW_KS" + command: + - /tmp/helm-tests.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: pod-etc-ceph + mountPath: /etc/ceph + - name: ceph-rgw-bin + mountPath: /tmp/helm-tests.sh + subPath: helm-tests.sh + readOnly: true + - name: ceph-keyring + mountPath: /tmp/client-keyring + subPath: key + readOnly: true + - name: ceph-rgw-etc + mountPath: /etc/ceph/ceph.conf + subPath: ceph.conf + readOnly: true +{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.object_store.api.internal | include "helm-toolkit.snippets.tls_volume_mount" | indent 8 }} +{{- end }} +{{ if .Values.conf.rgw_s3.enabled }} + - name: ceph-rgw-s3-validation +{{ tuple $envAll "ceph_rgw" | include "helm-toolkit.snippets.image" | indent 6 }} +{{ tuple $envAll $envAll.Values.pod.resources.tests | include "helm-toolkit.snippets.kubernetes_resources" | indent 6 }} +{{ dict "envAll" $envAll "application" "rgw_test" "container" "ceph_rgw_s3_validation" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 6 }} + env: +{{- with $env := dict "s3AdminSecret" $envAll.Values.secrets.rgw_s3.admin }} +{{- include "helm-toolkit.snippets.rgw_s3_admin_env_vars" $env | indent 8 }} +{{- end }} + - name: RGW_HOST + value: {{ tuple "ceph_object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} + - name: "RGW_TEST_TYPE" + value: "RGW_S3" + command: + - /tmp/helm-tests.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: pod-etc-ceph + mountPath: /etc/ceph + - name: ceph-rgw-bin + mountPath: /tmp/helm-tests.sh + subPath: helm-tests.sh + readOnly: true +{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.ceph_object_store.api.internal "path" "/etc/tls" | include "helm-toolkit.snippets.tls_volume_mount" | indent 8 }} +{{- end }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: pod-etc-ceph + emptyDir: {} + - name: ceph-rgw-bin + configMap: + name: ceph-rgw-bin + defaultMode: 0555 + - name: ceph-keyring + secret: + secretName: {{ .Values.secrets.keyrings.admin | quote }} + - name: ceph-rgw-etc + configMap: + name: ceph-rgw-etc + defaultMode: 0444 +{{- if .Values.conf.rgw_ks.enabled }} +{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.object_store.api.internal | include "helm-toolkit.snippets.tls_volume" | indent 4 }} +{{- end }} +{{- if .Values.conf.rgw_s3.enabled }} +{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.ceph_object_store.api.internal | include "helm-toolkit.snippets.tls_volume" | indent 4 }} +{{- end }} +{{- end }} diff --git a/ceph-rgw/templates/secret-ingress-tls.yaml b/ceph-rgw/templates/secret-ingress-tls.yaml new file mode 100644 index 0000000000..d9e46eb464 --- /dev/null +++ b/ceph-rgw/templates/secret-ingress-tls.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_ingress_tls ( and .Values.deployment.ceph .Values.conf.features.rgw ) }} +{{- include "helm-toolkit.manifests.secret_ingress_tls" ( dict "envAll" . "backendServiceType" "object_store" ) }} +{{- end }} diff --git a/ceph-rgw/templates/secret-keystone-rgw.yaml b/ceph-rgw/templates/secret-keystone-rgw.yaml new file mode 100644 index 0000000000..bf0ff156cd --- /dev/null +++ b/ceph-rgw/templates/secret-keystone-rgw.yaml @@ -0,0 +1,33 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_keystone_rgw .Values.deployment.ceph }} +{{- $envAll := . }} +{{- range $key1, $userClass := tuple "swift" }} +{{- $secretName := index $envAll.Values.secrets.identity "user_rgw" }} +{{- $auth := index $envAll.Values.endpoints.identity.auth $userClass }} +{{ $osAuthType := $auth.os_auth_type }} +{{ $osTenantName := $auth.os_tenant_name }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} +type: Opaque +data: +{{- tuple $userClass "internal" $envAll | include "helm-toolkit.snippets.keystone_secret_openrc" | indent 2 }} + OS_AUTH_TYPE: {{ $osAuthType | b64enc }} + OS_TENANT_NAME: {{ $osTenantName | b64enc }} +{{ end }} +{{- end }} diff --git a/ceph-rgw/templates/secret-keystone.yaml b/ceph-rgw/templates/secret-keystone.yaml new file mode 100644 index 0000000000..eac2d05c94 --- /dev/null +++ b/ceph-rgw/templates/secret-keystone.yaml @@ -0,0 +1,28 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_keystone .Values.conf.rgw_ks.enabled }} +{{- $envAll := . }} +{{- range $key1, $userClass := tuple "admin" "swift" }} +{{- $secretName := index $envAll.Values.secrets.identity $userClass }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} +type: Opaque +data: +{{- tuple $userClass "internal" $envAll | include "helm-toolkit.snippets.keystone_secret_openrc" | indent 2 -}} +{{- end }} +{{- end }} diff --git a/ceph-rgw/templates/secret-registry.yaml b/ceph-rgw/templates/secret-registry.yaml new file mode 100644 index 0000000000..da979b3223 --- /dev/null +++ b/ceph-rgw/templates/secret-registry.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_registry .Values.endpoints.oci_image_registry.auth.enabled }} +{{ include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) }} +{{- end }} diff --git a/ceph-rgw/templates/secret-s3-rgw.yaml b/ceph-rgw/templates/secret-s3-rgw.yaml new file mode 100644 index 0000000000..a732eab3e2 --- /dev/null +++ b/ceph-rgw/templates/secret-s3-rgw.yaml @@ -0,0 +1,28 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_s3_rgw }} +{{- $envAll := . }} +{{- $secretName := index $envAll.Values.secrets.rgw_s3.admin }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} +type: Opaque +data: + S3_ADMIN_USERNAME: {{ .Values.endpoints.ceph_object_store.auth.admin.username | b64enc }} + S3_ADMIN_ACCESS_KEY: {{ .Values.endpoints.ceph_object_store.auth.admin.access_key | b64enc }} + S3_ADMIN_SECRET_KEY: {{ .Values.endpoints.ceph_object_store.auth.admin.secret_key | b64enc }} +{{- end }} diff --git a/ceph-rgw/templates/service-ingress-rgw.yaml b/ceph-rgw/templates/service-ingress-rgw.yaml new file mode 100644 index 0000000000..9a9bd4d602 --- /dev/null +++ b/ceph-rgw/templates/service-ingress-rgw.yaml @@ -0,0 +1,23 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{ $object_store_name := "object_store" }} +{{- if .Values.conf.rgw_s3.enabled }} +{{ $object_store_name = "ceph_object_store" }} +{{- end }} + +{{- if and .Values.manifests.service_ingress_rgw ( and .Values.deployment.ceph (and .Values.network.api.ingress.public .Values.conf.features.rgw ) ) }} +{{- $serviceIngressOpts := dict "envAll" . "backendServiceType" $object_store_name -}} +{{ $serviceIngressOpts | include "helm-toolkit.manifests.service_ingress" }} +{{- end }} diff --git a/ceph-rgw/templates/service-rgw.yaml b/ceph-rgw/templates/service-rgw.yaml new file mode 100644 index 0000000000..fcb236ef92 --- /dev/null +++ b/ceph-rgw/templates/service-rgw.yaml @@ -0,0 +1,43 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.service_rgw ( and .Values.deployment.ceph .Values.conf.features.rgw ) }} +{{- $envAll := . }} +{{ $object_store_name := "object_store" }} +{{- if .Values.conf.rgw_s3.enabled }} +{{ $object_store_name = "ceph_object_store" }} +{{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: ceph-rgw +spec: + ports: + - name: ceph-rgw + port: {{ tuple $object_store_name "internal" "api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + protocol: TCP + targetPort: {{ tuple $object_store_name "internal" "api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + {{ if .Values.network.api.node_port.enabled }} + nodePort: {{ .Values.network.api.node_port.port }} + {{ end }} + selector: +{{ tuple $envAll "ceph" "rgw" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + {{ if .Values.network.api.node_port.enabled }} + type: NodePort + {{ if .Values.network.api.external_policy_local }} + externalTrafficPolicy: Local + {{ end }} + {{ end }} +{{- end }} diff --git a/ceph-rgw/values.yaml b/ceph-rgw/values.yaml new file mode 100644 index 0000000000..176f39eeec --- /dev/null +++ b/ceph-rgw/values.yaml @@ -0,0 +1,745 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for ceph-client. +# This is a YAML-formatted file. +# Declare name/value pairs to be passed into your templates. +# name: value + +--- +deployment: + ceph: false + +release_group: null + +images: + pull_policy: IfNotPresent + tags: + ceph_bootstrap: 'docker.io/openstackhelm/ceph-daemon:ubuntu_jammy_19.2.1-1-20250207' + ceph_config_helper: 'docker.io/openstackhelm/ceph-config-helper:ubuntu_jammy_19.2.1-1-20250207' + ceph_rgw: 'docker.io/openstackhelm/ceph-daemon:ubuntu_jammy_19.2.1-1-20250207' + ceph_rgw_pool: 'docker.io/openstackhelm/ceph-config-helper:ubuntu_jammy_19.2.1-1-20250207' + dep_check: 'quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal' + image_repo_sync: 'docker.io/library/docker:17.07.0' + rgw_s3_admin: 'docker.io/openstackhelm/ceph-config-helper:ubuntu_jammy_19.2.1-1-20250207' + rgw_placement_targets: 'docker.io/openstackhelm/ceph-config-helper:ubuntu_jammy_19.2.1-1-20250207' + ks_endpoints: 'docker.io/openstackhelm/heat:2024.1-ubuntu_jammy' + ks_service: 'docker.io/openstackhelm/heat:2024.1-ubuntu_jammy' + ks_user: 'docker.io/openstackhelm/heat:2024.1-ubuntu_jammy' + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +labels: + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + test: + node_selector_key: openstack-control-plane + node_selector_value: enabled + rgw: + node_selector_key: ceph-rgw + node_selector_value: enabled + +pod: + security_context: + rgw: + pod: + runAsUser: 64045 + container: + init_dirs: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + rgw_init: + runAsUser: 0 + readOnlyRootFilesystem: true + rgw: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + rgw_storage_init: + pod: + runAsUser: 64045 + container: + keyring_placement: + runAsUser: 0 + readOnlyRootFilesystem: true + rgw_storage_init: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + rgw_restart: + pod: + runAsUser: 65534 + container: + ceph-rgw-restart: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + rgw_s3_admin: + pod: + runAsUser: 64045 + container: + keyring_placement: + runAsUser: 0 + readOnlyRootFilesystem: true + create_s3_admin: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + rgw_placement_targets: + pod: + runAsUser: 64045 + container: + keyring_placement: + runAsUser: 0 + readOnlyRootFilesystem: true + create_rgw_placement_targets: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + rgw_test: + pod: + runAsUser: 64045 + rgw_test: + ceph_rgw_ks_validation: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + ceph_rgw_s3_validation: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + bootstrap: + pod: + runAsUser: 65534 + container: + keyring_placement: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + bootstrap: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + rgw_pool: + pod: + runAsUser: 65534 + container: + rgw_pool: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + dns_policy: "ClusterFirstWithHostNet" + replicas: + rgw: 2 + lifecycle: + upgrades: + deployments: + pod_replacement_strategy: RollingUpdate + revision_history: 3 + rolling_update: + max_surge: 50% + max_unavailable: 50% + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + weight: + default: 10 + resources: + enabled: false + rgw: + requests: + memory: "128Mi" + cpu: "250m" + limits: + memory: "512Mi" + cpu: "1000m" + jobs: + bootstrap: + limits: + memory: "1024Mi" + cpu: "2000m" + requests: + memory: "128Mi" + cpu: "500m" + ceph-rgw-storage-init: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + image_repo_sync: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + ks-endpoints: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + ks_service: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + ks_user: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + rgw_s3_admin: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + rgw_placement_targets: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + rgw_restart: + limits: + memory: "1024Mi" + cpu: "2000m" + requests: + memory: "128Mi" + cpu: "500m" + rgw_pool: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + tests: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + tolerations: + rgw: + tolerations: + - effect: NoExecute + key: node.kubernetes.io/not-ready + operator: Exists + tolerationSeconds: 60 + - effect: NoExecute + key: node.kubernetes.io/unreachable + operator: Exists + tolerationSeconds: 60 + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + probes: + api: + ceph-rgw: + readiness: + enabled: true + params: + timeoutSeconds: 5 + liveness: + enabled: true + params: + initialDelaySeconds: 120 + timeoutSeconds: 5 + +network_policy: + rgw: + ingress: + - {} + egress: + - {} + +ceph_client: + configmap: ceph-etc + +secrets: + keyrings: + mon: ceph-mon-keyring + mds: ceph-bootstrap-mds-keyring + osd: ceph-bootstrap-osd-keyring + rgw: os-ceph-bootstrap-rgw-keyring + mgr: ceph-bootstrap-mgr-keyring + admin: pvc-ceph-client-key + identity: + admin: ceph-keystone-admin + swift: ceph-keystone-user + user_rgw: ceph-keystone-user-rgw + oci_image_registry: + ceph-rgw: ceph-rgw-oci-image-registry-key + rgw_s3: + admin: radosgw-s3-admin-creds + tls: + object_store: + api: + public: ceph-tls-public + internal: ceph-rgw-ks-tls-api + keystone: keystone-tls-api + ceph_object_store: + api: + public: ceph-rgw-s3-tls-public + internal: ceph-rgw-s3-tls-api + +network: + api: + ingress: + public: true + classes: + namespace: "nginx" + cluster: "nginx-cluster" + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / + nginx.ingress.kubernetes.io/proxy-body-size: "0" + nginx.ingress.kubernetes.io/proxy-max-temp-file-size: "0" + external_policy_local: false + node_port: + enabled: false + port: 30004 + public: 192.168.0.0/16 + cluster: 192.168.0.0/16 + +conf: + templates: + keyring: + admin: | + [client.admin] + key = {{ key }} + auid = 0 + caps mds = "allow" + caps mon = "allow *" + caps osd = "allow *" + caps mgr = "allow *" + bootstrap: + rgw: | + [client.bootstrap-rgw] + key = {{ key }} + caps mgr = "allow profile bootstrap-rgw" + features: + rgw: true + pool: + # NOTE(portdirect): this drives a simple approximation of + # https://ceph.com/pgcalc/, the `target.osd` key should be set to match the + # expected number of osds in a cluster, and the `target.pg_per_osd` should be + # set to match the desired number of placement groups on each OSD. + crush: + # NOTE(portdirect): to use RBD devices with Ubuntu 16.04's 4.4.x series + # kernel this should be set to `hammer` + tunables: null + target: + # NOTE(portdirect): arbitrarily we set the default number of expected OSD's to 5 + # to match the number of nodes in the OSH gate. + osd: 5 + pg_per_osd: 100 + default: + # NOTE(portdirect): this should be 'same_host' for a single node + # cluster to be in a healthy state + crush_rule: replicated_rule + # NOTE(portdirect): this section describes the pools that will be managed by + # the ceph pool management job, as it tunes the pgs and crush rule, based on + # the above. + spec: + # RBD pool + - name: rbd + application: rbd + replication: 3 + percent_total_data: 40 + # CephFS pools + - name: cephfs_metadata + application: cephfs + replication: 3 + percent_total_data: 5 + - name: cephfs_data + application: cephfs + replication: 3 + percent_total_data: 10 + # RadosGW pools + - name: .rgw.root + application: rgw + replication: 3 + percent_total_data: 0.1 + - name: default.rgw.control + application: rgw + replication: 3 + percent_total_data: 0.1 + - name: default.rgw.data.root + application: rgw + replication: 3 + percent_total_data: 0.1 + - name: default.rgw.gc + application: rgw + replication: 3 + percent_total_data: 0.1 + - name: default.rgw.log + application: rgw + replication: 3 + percent_total_data: 0.1 + - name: default.rgw.intent-log + application: rgw + replication: 3 + percent_total_data: 0.1 + - name: default.rgw.meta + application: rgw + replication: 3 + percent_total_data: 0.1 + - name: default.rgw.usage + application: rgw + replication: 3 + percent_total_data: 0.1 + - name: default.rgw.users.keys + application: rgw + replication: 3 + percent_total_data: 0.1 + - name: default.rgw.users.email + application: rgw + replication: 3 + percent_total_data: 0.1 + - name: default.rgw.users.swift + application: rgw + replication: 3 + percent_total_data: 0.1 + - name: default.rgw.users.uid + application: rgw + replication: 3 + percent_total_data: 0.1 + - name: default.rgw.buckets.extra + application: rgw + replication: 3 + percent_total_data: 0.1 + - name: default.rgw.buckets.index + application: rgw + replication: 3 + percent_total_data: 3 + - name: default.rgw.buckets.data + application: rgw + replication: 3 + percent_total_data: 34.8 + rgw_placement_targets: + - name: default-placement + data_pool: default.rgw.buckets.data + # Set 'delete' to true to delete an existing placement target. A + # non-existent placement target will be created and deleted in a single + # step. + # delete: true + rgw: + config: + # NOTE (portdirect): See http://tracker.ceph.com/issues/21226 + rgw_keystone_token_cache_size: 0 + # NOTE (JCL): See http://tracker.ceph.com/issues/7073 + rgw_gc_max_objs: 997 + # NOTE (JCL): See http://tracker.ceph.com/issues/24937 + # NOTE (JCL): See https://tracker.ceph.com/issues/24551 + rgw_dynamic_resharding: false + rgw_num_rados_handles: 4 + rgw_override_bucket_index_max_shards: 8 + rgw_restart: + timeout: 600 + rgw_ks: + enabled: false + config: + rgw_keystone_api_version: 3 + rgw_keystone_accepted_roles: "admin, member" + rgw_keystone_implicit_tenants: true + rgw_keystone_make_new_tenants: true + rgw_s3_auth_use_keystone: true + rgw_s3_auth_order: "local, external, sts" + rgw_swift_account_in_url: true + rgw_swift_url: null + rgw_s3: + enabled: false + admin_caps: "users=*;buckets=*;zone=*" + config: + # NOTE (supamatt): Unfortunately we do not conform to S3 compliant names with some of our charts + rgw_relaxed_s3_bucket_names: true + ceph: + global: + # auth + cephx: true + cephx_require_signatures: false + cephx_cluster_require_signatures: true + cephx_service_require_signatures: false + objecter_inflight_op_bytes: "1073741824" + debug_ms: "0/0" + log_file: /dev/stdout + mon_cluster_log_file: /dev/stdout + # CNTT certification required fields + rgw_max_attr_name_len: 64 + rgw_max_attrs_num_in_req: 32 + rgw_max_attr_size: 1024 + rgw_swift_versioning_enabled: true + osd: + osd_mkfs_type: xfs + osd_mkfs_options_xfs: -f -i size=2048 + osd_max_object_name_len: 256 + ms_bind_port_min: 6800 + ms_bind_port_max: 7100 + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - ceph-rgw-image-repo-sync + services: + - endpoint: node + service: local_image_registry + targeted: + keystone: + rgw: + services: + - endpoint: internal + service: identity + s3: + rgw: {} + static: + rgw: + jobs: + - ceph-rgw-storage-init + rgw_restart: + services: + - endpoint: internal + service: ceph_object_store + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + ks_endpoints: + jobs: + - ceph-ks-service + services: + - endpoint: internal + service: identity + ks_service: + services: + - endpoint: internal + service: identity + ks_user: + services: + - endpoint: internal + service: identity + rgw_s3_admin: + services: + - endpoint: internal + service: ceph_object_store + rgw_placement_targets: + services: + - endpoint: internal + service: ceph_object_store + rgw_pool: + jobs: + - ceph-rgw-storage-init + tests: + services: + - endpoint: internal + service: ceph_object_store + +bootstrap: + enabled: false + script: | + ceph -s + function ensure_pool () { + ceph osd pool stats $1 || ceph osd pool create $1 $2 + if [[ $(ceph mon versions | awk '/version/{print $3}' | cut -d. -f1) -ge 12 ]]; then + ceph osd pool application enable $1 $3 + fi + } + #ensure_pool volumes 8 cinder + + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + ceph-rgw: + username: ceph-rgw + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + identity: + name: keystone + namespace: null + auth: + admin: + region_name: RegionOne + username: admin + password: password + project_name: admin + user_domain_name: default + project_domain_name: default + os_auth_type: password + os_tenant_name: admin + swift: + role: admin + region_name: RegionOne + username: swift + password: password + project_name: service + user_domain_name: service + project_domain_name: service + os_auth_type: password + os_tenant_name: admin + hosts: + default: keystone + internal: keystone-api + host_fqdn_override: + default: null + path: + default: /v3 + scheme: + default: http + port: + api: + default: 80 + internal: 5000 + object_store: + name: swift + namespace: null + hosts: + default: ceph-rgw + public: radosgw + host_fqdn_override: + default: null + # NOTE(portdirect): this chart supports TLS for fqdn over-ridden public + # endpoints using the following format: + # public: + # host: null + # tls: + # crt: null + # key: null + path: + default: /swift/v1/KEY_$(tenant_id)s + scheme: + default: http + port: + api: + default: 8088 + public: 80 + ceph_object_store: + name: radosgw + namespace: null + auth: + admin: + # NOTE(srwilkers): These defaults should be used for testing only, and + # should be changed before deploying to production + username: s3_admin + access_key: "admin_access_key" + secret_key: "admin_secret_key" + hosts: + default: ceph-rgw + public: radosgw + host_fqdn_override: + default: null + path: + default: null + scheme: + default: http + port: + api: + default: 8088 + public: 80 + ceph_mon: + namespace: null + hosts: + default: ceph-mon + discovery: ceph-mon-discovery + host_fqdn_override: + default: null + port: + mon: + default: 6789 + mon_msgr2: + default: 3300 + + kube_dns: + namespace: kube-system + name: kubernetes-dns + hosts: + default: kube-dns + host_fqdn_override: + default: null + path: + default: null + scheme: http + port: + dns_tcp: + default: 53 + dns: + default: 53 + protocol: UDP + +jobs: + rgw_pool: + restartPolicy: OnFailure + +manifests: + certificates: false + configmap_ceph_templates: true + configmap_bin: true + configmap_bin_ks: true + configmap_test_bin: true + configmap_etc: true + deployment_rgw: true + ingress_rgw: true + job_bootstrap: false + job_rgw_restart: false + job_ceph_rgw_storage_init: true + job_image_repo_sync: true + job_ks_endpoints: true + job_ks_service: true + job_ks_user: true + job_s3_admin: true + job_rgw_placement_targets: false + job_rgw_pool: true + secret_s3_rgw: true + secret_keystone_rgw: true + secret_ingress_tls: true + secret_keystone: true + secret_registry: true + service_ingress_rgw: true + service_rgw: true + helm_tests: true + network_policy: false +... diff --git a/cert-rotation/Chart.yaml b/cert-rotation/Chart.yaml new file mode 100644 index 0000000000..c97226c42e --- /dev/null +++ b/cert-rotation/Chart.yaml @@ -0,0 +1,24 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: "1.0" +description: Rotate the certificates generated by cert-manager +home: https://cert-manager.io/ +name: cert-rotation +version: 2024.2.0 +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit + version: ">= 0.1.0" +... diff --git a/cert-rotation/templates/bin/_rotate-certs.sh.tpl b/cert-rotation/templates/bin/_rotate-certs.sh.tpl new file mode 100644 index 0000000000..fe55d7bac6 --- /dev/null +++ b/cert-rotation/templates/bin/_rotate-certs.sh.tpl @@ -0,0 +1,220 @@ +#!/bin/bash + +set -x + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + + +COMMAND="${@:-rotate_job}" + +namespace={{ .Release.Namespace }} +minDaysToExpiry={{ .Values.jobs.rotate.max_days_to_expiry }} + +rotateBefore=$(($(date +%s) + (86400*$minDaysToExpiry))) + +function rotate_and_get_certs_list(){ + # Rotate the certificates if the expiry date of certificates is within the + # max_days_to_expiry days + + # List of secret and certificates rotated + local -n secRotated=$1 + deleteAllSecrets=$2 + certRotated=() + + for certificate in $(kubectl get certificates -n ${namespace} --no-headers | awk '{ print $1 }') + do + certInfo=($(kubectl get certificate -n ${namespace} ${certificate} -o json | jq -r '.spec["secretName"],.status["notAfter"]')) + secretName=${certInfo[0]} + notAfter=$(date -d"${certInfo[1]}" '+%s') + deleteSecret=false + if ${deleteAllSecrets} || [ ${rotateBefore} -gt ${notAfter} ] + then + # Rotate the certificates/secrets and add to list. + echo "Deleting secret: ${secretName}" + kubectl delete secret -n ${namespace} $secretName + secRotated+=(${secretName}) + certRotated+=(${certificate}) + fi + done + + # Ensure certificates are re-issued + if [ ! -z ${certRotated} ] + then + for cert in ${certRotated[@]} + do + counter=0 + retried=false + while [ "$(kubectl get certificate -n ${namespace} ${cert} -o json | jq -r '.status.conditions[].status')" != "True" ] + do + # Wait for secret to become ready. Wait for 300 seconds maximum. Sleep for 10 seconds + if [ ${counter} -ge 30 ] + then + # Seems certificate is not in ready state yet, may be there is an issue be renewing the certificate. + # Try one more time before failing it. The name of the secret would be different at this time (when in + # process of issuing) + priSeckeyName=$(kubectl get certificate -n ${namespace} ${cert} -o json | jq -r '.status["nextPrivateKeySecretName"]') + + if [ ${retried} = false ] && [ ! -z ${priSeckeyName} ] + then + echo "Deleting interim failed secret ${priSeckeyName} in namespace ${namespace}" + kubectl delete secret -n ${namespace} ${priSeckeyName} + retried=true + counter=0 + else + # Tried 2 times to renew the certificate, something is not right. Log error and + # continue to check the status of next certificate. Once the status of all the + # certificates has been checked, the pods need to be restarted so that the successfully + # renewed certificates can be deployed. + echo "ERROR: Rotated certificate ${cert} in ${namespace} is not ready." + break + fi + fi + echo "Rotated certificate ${cert} in ${namespace} is not ready yet ... waiting" + counter=$((counter+1)) + sleep 10 + done + + done + fi +} + +function get_cert_list_rotated_by_cert_manager_rotate(){ + + local -n secRotated=$1 + + # Get the time when the last cron job was run successfully + lastCronTime=$(kubectl get jobs -n ${namespace} --no-headers -l application=cert-manager,component=cert-rotate -o json | jq -r '.items[] | select(.status.succeeded != null) | .status.completionTime' | sort -r | head -n 1) + + if [ ! -z ${lastCronTime} ] + then + lastCronTimeSec=$(date -d"${lastCronTime}" '+%s') + + for certificate in $(kubectl get certificates -n ${namespace} --no-headers | awk '{ print $1 }') + do + certInfo=($(kubectl get certificate -n ${namespace} ${certificate} -o json | jq -r '.spec["secretName"],.status["notBefore"]')) + secretName=${certInfo[0]} + notBefore=$(date -d"${certInfo[1]}" '+%s') + + # if the certificate was created after last cronjob run means it was + # rotated by the cert-manager, add to the list. + if [[ ${notBefore} -gt ${lastCronTimeSec} ]] + then + secRotated+=(${secretName}) + fi + done + fi +} + +function restart_the_pods(){ + + local -n secRotated=$1 + + if [ -z ${secRotated} ] + then + echo "All certificates are still valid in ${namespace} namespace. No pod needs restart" + exit 0 + fi + + # Restart the pods using kubernetes rollout restart. This will restarts the applications + # with zero downtime. + for kind in statefulset deployment daemonset + do + # Need to find which kinds mounts the secret that has been rotated. To do this + # for a kind (statefulset, deployment, or daemonset) + # - get the name of the kind (which will index 1 = idx=0 of the output) + # - get the names of the secrets mounted on this kind (which will be index 2 = idx+1) + # - find if tls.crt was mounted to the container: get the subpaths of volumeMount in + # the container and grep for tls.crt. (This will be index 3 = idx+2) + # - or, find if tls.crt was mounted to the initContainer (This will be index 4 = idx+3) + + resource=($(kubectl get ${kind} -n ${namespace} -o custom-columns='NAME:.metadata.name,SECRETS:.spec.template.spec.volumes[*].secret.secretName,TLS-CONTAINER:.spec.template.spec.containers[*].volumeMounts[*].subPath,TLS-INIT:.spec.template.spec.initContainers[*].volumeMounts[*].subPath' --no-headers | grep tls.crt || true)) + + idx=0 + while [[ $idx -lt ${#resource[@]} ]] + do + # Name of the kind + resourceName=${resource[$idx]} + + # List of secrets mounted to this kind + resourceSecrets=${resource[$idx+1]} + + # For each secret mounted to this kind, check if it was rotated (present in + # the list secRotated) and if it was, then trigger rolling restart for this kind. + for secret in ${resourceSecrets//,/ } + do + if [[ "${secRotated[@]}" =~ "${secret}" ]] + then + echo "Restarting ${kind} ${resourceName} in ${namespace} namespace." + kubectl rollout restart -n ${namespace} ${kind} ${resourceName} + break + fi + done + + # Since we have 4 custom columns in the output, every 5th index will be start of new tuple. + # Jump to the next tuple. + idx=$((idx+4)) + done + done +} + +function rotate_cron(){ + # Rotate cronjob invoked this script. + # 1. If the expiry date of certificates is within the max_days_to_expiry days + # the rotate the certificates and restart the pods + # 2. Else if the certificates were rotated by cert-manager, then restart + # the pods. + + secretsRotated=() + deleteAllSecrets=false + + rotate_and_get_certs_list secretsRotated $deleteAllSecrets + + if [ ! -z ${secretsRotated} ] + then + # Certs rotated, restart pods + restart_the_pods secretsRotated + else + # Check if the certificates were rotated by the cert-manager and get the list of + # rotated certificates so that the corresponding pods can be restarted + get_cert_list_rotated_by_cert_manager_rotate secretsRotated + if [ ! -z ${secretsRotated} ] + then + restart_the_pods secretsRotated + else + echo "All certificates are still valid in ${namespace} namespace" + fi + fi +} + +function rotate_job(){ + # Rotate job invoked this script. + # 1. Rotate all certificates by deleting the secrets and restart the pods + + secretsRotated=() + deleteAllSecrets=true + + rotate_and_get_certs_list secretsRotated $deleteAllSecrets + + if [ ! -z ${secretsRotated} ] + then + # Certs rotated, restart pods + restart_the_pods secretsRotated + else + echo "All certificates are still valid in ${namespace} namespace" + fi +} + +$COMMAND +exit 0 diff --git a/cert-rotation/templates/configmap-bin.yaml b/cert-rotation/templates/configmap-bin.yaml new file mode 100644 index 0000000000..e13463a6ac --- /dev/null +++ b/cert-rotation/templates/configmap-bin.yaml @@ -0,0 +1,25 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_bin }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: cert-rotate-bin +data: + rotate-certs.sh: | +{{ tuple "bin/_rotate-certs.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{ end }} diff --git a/cert-rotation/templates/cron-job-cert-rotate.yaml b/cert-rotation/templates/cron-job-cert-rotate.yaml new file mode 100644 index 0000000000..92377a9ad1 --- /dev/null +++ b/cert-rotation/templates/cron-job-cert-rotate.yaml @@ -0,0 +1,120 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.cron_job_cert_rotate}} +{{- $envAll := . }} + +{{- $serviceAccountName := "cert-rotate-cron" }} +{{ tuple $envAll "cert_rotate" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $serviceAccountName }} +rules: + - apiGroups: + - cert-manager.io + resources: + - certificates + verbs: + - get + - list + - update + - patch + - apiGroups: + - "*" + resources: + - pods + - secrets + - jobs + - statefulsets + - daemonsets + - deployments + verbs: + - get + - list + - update + - patch + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $serviceAccountName }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: cert-rotate + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "cert-manager" "cert-rotate-cron" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + suspend: {{ .Values.jobs.rotate.suspend }} + schedule: {{ .Values.jobs.rotate.cron | quote }} + successfulJobsHistoryLimit: {{ .Values.jobs.rotate.history.success }} + failedJobsHistoryLimit: {{ .Values.jobs.rotate.history.failed }} +{{- if .Values.jobs.rotate.starting_deadline }} + startingDeadlineSeconds: {{ .Values.jobs.rotate.starting_deadline }} +{{- end }} + concurrencyPolicy: Forbid + jobTemplate: + metadata: + labels: +{{ tuple $envAll "cert-manager" "cert-rotate" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + spec: + template: + metadata: + labels: +{{ tuple $envAll "cert-manager" "cert-rotate" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 12 }} + spec: + serviceAccountName: {{ $serviceAccountName }} +{{ dict "envAll" $envAll "application" "cert_rotate" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 10 }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} + initContainers: +{{ tuple $envAll "cert_rotate" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 12 }} + containers: + - name: cert-rotate +{{ tuple $envAll "cert_rotation" | include "helm-toolkit.snippets.image" | indent 14 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.cert_rotate | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }} +{{ dict "envAll" $envAll "application" "cert_rotate" "container" "cert_rotate" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 14 }} + command: + - /tmp/rotate-certs.sh + - rotate_cron + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: cert-rotate-bin + mountPath: /tmp/rotate-certs.sh + subPath: rotate-certs.sh + readOnly: true + volumes: + - name: pod-tmp + emptyDir: {} + - name: cert-rotate-bin + configMap: + name: cert-rotate-bin + defaultMode: 0555 +{{- end }} diff --git a/cert-rotation/templates/job-cert-rotate.yaml b/cert-rotation/templates/job-cert-rotate.yaml new file mode 100644 index 0000000000..f508a7d9d2 --- /dev/null +++ b/cert-rotation/templates/job-cert-rotate.yaml @@ -0,0 +1,107 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.job_cert_rotate}} +{{- $envAll := . }} + +{{- $serviceAccountName := "cert-rotate-job" }} +{{ tuple $envAll "cert_rotate" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $serviceAccountName }} +rules: + - apiGroups: + - cert-manager.io + resources: + - certificates + verbs: + - get + - list + - update + - patch + - apiGroups: + - "*" + resources: + - pods + - secrets + - jobs + - statefulsets + - daemonsets + - deployments + verbs: + - get + - list + - update + - patch + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $serviceAccountName }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: cert-rotate-job + labels: +{{ tuple $envAll "cert-manager" "cert-rotate-job" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + template: + metadata: + labels: +{{ tuple $envAll "cert-manager" "cert-rotate" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + spec: + serviceAccountName: {{ $serviceAccountName }} +{{ dict "envAll" $envAll "application" "cert_rotate" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} + initContainers: +{{ tuple $envAll "cert_rotate" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 12 }} + containers: + - name: cert-rotate +{{ tuple $envAll "cert_rotation" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.cert_rotate | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "cert_rotate" "container" "cert_rotate" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/rotate-certs.sh + - rotate_job + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: cert-rotate-bin + mountPath: /tmp/rotate-certs.sh + subPath: rotate-certs.sh + readOnly: true + volumes: + - name: pod-tmp + emptyDir: {} + - name: cert-rotate-bin + configMap: + name: cert-rotate-bin + defaultMode: 0555 +{{- end }} diff --git a/cert-rotation/templates/secret-registry.yaml b/cert-rotation/templates/secret-registry.yaml new file mode 100644 index 0000000000..da979b3223 --- /dev/null +++ b/cert-rotation/templates/secret-registry.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_registry .Values.endpoints.oci_image_registry.auth.enabled }} +{{ include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) }} +{{- end }} diff --git a/cert-rotation/values.yaml b/cert-rotation/values.yaml new file mode 100644 index 0000000000..25fa102e2b --- /dev/null +++ b/cert-rotation/values.yaml @@ -0,0 +1,82 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- + +images: + tags: + cert_rotation: 'docker.io/openstackhelm/ceph-config-helper:latest-ubuntu_jammy' + dep_check: 'quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal' + local_registry: + active: false +labels: + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled +jobs: + rotate: + # Run at 1:00AM on 1st of each month + cron: "0 1 1 * *" + starting_deadline: 600 + history: + success: 3 + failed: 1 + # Number of day before expiry should certs be rotated. + max_days_to_expiry: 45 + suspend: false +pod: + security_context: + cert_rotate: + pod: + runAsUser: 42424 + container: + cert_rotate: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + resources: + enabled: false + jobs: + cert_rotate: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" +dependencies: + static: + cert_rotate: null +secrets: + oci_image_registry: + cert-rotation: cert-rotation-oci-image-registry-key +endpoints: + cluster_domain_suffix: cluster.local + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + cert-rotation: + username: cert-rotation + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null +manifests: + configmap_bin: true + cron_job_cert_rotate: false + job_cert_rotate: false + secret_registry: true +... diff --git a/daemonjob-controller/Chart.yaml b/daemonjob-controller/Chart.yaml new file mode 100644 index 0000000000..46a952a0ae --- /dev/null +++ b/daemonjob-controller/Chart.yaml @@ -0,0 +1,24 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: v1.0.0 +description: A Helm chart for DaemonjobController +name: daemonjob-controller +version: 2024.2.0 +home: https://opendev.org/openstack +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit + version: ">= 0.1.0" +... diff --git a/daemonjob-controller/templates/bin/_sync-hook.py.tpl b/daemonjob-controller/templates/bin/_sync-hook.py.tpl new file mode 100644 index 0000000000..546f0dd061 --- /dev/null +++ b/daemonjob-controller/templates/bin/_sync-hook.py.tpl @@ -0,0 +1,106 @@ +#!/usr/bin/env python +{{/* +Copyright 2019 Google Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +import copy +from http.server import BaseHTTPRequestHandler, HTTPServer +import io +import json + + +def is_job_finished(job): + if 'status' in job: + desiredNumberScheduled = job['status'].get('desiredNumberScheduled', 1) + numberReady = job['status'].get('numberReady', 0) + if (desiredNumberScheduled == numberReady and + desiredNumberScheduled > 0): + return True + return False + + +def new_daemon(job): + pause_image = {{ .Values.images.tags.pause | quote }} + daemon = copy.deepcopy(job) + daemon['apiVersion'] = 'apps/v1' + daemon['kind'] = 'DaemonSet' + daemon['metadata'] = {} + daemon['metadata']['name'] = '%s-dj' % (job['metadata']['name']) + daemon['metadata']['labels'] = copy.deepcopy( + job['spec']['template']['metadata']['labels']) + daemon['spec'] = {} + daemon['spec']['template'] = copy.deepcopy(job['spec']['template']) + daemon['spec']['template']['spec']['initContainers'] = copy.deepcopy( + job['spec']['template']['spec']['containers']) + daemon['spec']['template']['spec']['containers'] = [ + {'name': "pause", 'image': job['spec'].get( + 'pauseImage', pause_image), + 'resources': {'requests': {'cpu': '10m'}}}] + daemon['spec']['selector'] = {'matchLabels': copy.deepcopy( + job['spec']['template']['metadata']['labels'])} + + return daemon + + +class Controller(BaseHTTPRequestHandler): + def sync(self, job, children): + desired_status = {} + child = '%s-dj' % (job['metadata']['name']) + + # If the job already finished at some point, freeze the status, + # delete children, and take no further action. + if is_job_finished(job): + desired_status = copy.deepcopy(job['status']) + desired_status['conditions'] = [ + {'type': 'Complete', 'status': 'True'}] + return {'status': desired_status, 'children': []} + + # Compute status based on what we observed, + # before building desired state. + # Our .status is just a copy of the DaemonSet . + # status with extra fields. + desired_status = copy.deepcopy( + children['DaemonSet.apps/v1'].get(child, {}).get('status', {})) + if is_job_finished(children['DaemonSet.apps/v1'].get(child, {})): + desired_status['conditions'] = [ + {'type': 'Complete', 'status': 'True'}] + else: + desired_status['conditions'] = [ + {'type': 'Complete', 'status': 'False'}] + + # Always generate desired state for child if we reach this point. + # We should not delete children until after we know we've recorded + # completion in our status, which was the first check we did above. + desired_child = new_daemon(job) + return {'status': desired_status, 'children': [desired_child]} + + def do_POST(self): + observed = json.loads(self.rfile.read( + int(self.headers.get('Content-Length')))) + desired = self.sync(observed['parent'], observed['children']) + self.send_response(200) + self.send_header('Content-type', 'application/json') + self.end_headers() + out = io.TextIOWrapper( + self.wfile, + encoding='utf-8', + line_buffering=False, + write_through=True, + ) + out.write(json.dumps(desired)) + out.detach() + + +HTTPServer(('', 80), Controller).serve_forever() diff --git a/daemonjob-controller/templates/composite-controller.yaml b/daemonjob-controller/templates/composite-controller.yaml new file mode 100644 index 0000000000..b3a2523cae --- /dev/null +++ b/daemonjob-controller/templates/composite-controller.yaml @@ -0,0 +1,33 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{ $groupName := .Values.crds.group_name }} +{{ $groupVersion := .Values.crds.group_version }} +{{ $groupVersionFormat := printf "%s/%s" $groupName $groupVersion }} +apiVersion: metacontroller.k8s.io/v1alpha1 +kind: CompositeController +metadata: + name: daemonjob-controller +spec: + generateSelector: true + parentResource: + apiVersion: {{ $groupVersionFormat }} + resource: daemonjobs + childResources: + - apiVersion: apps/v1 + resource: daemonsets + hooks: + sync: + webhook: + url: http://daemonjob-controller.metacontroller/sync diff --git a/daemonjob-controller/templates/configmap-bin.yaml b/daemonjob-controller/templates/configmap-bin.yaml new file mode 100644 index 0000000000..01fd461f8a --- /dev/null +++ b/daemonjob-controller/templates/configmap-bin.yaml @@ -0,0 +1,25 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_bin }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: daemonjob-controller-bin + namespace: {{ .Release.Namespace }} +data: + sync.py: | +{{ tuple "bin/_sync-hook.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} diff --git a/daemonjob-controller/templates/crd.yaml b/daemonjob-controller/templates/crd.yaml new file mode 100644 index 0000000000..7e44cfa0e7 --- /dev/null +++ b/daemonjob-controller/templates/crd.yaml @@ -0,0 +1,4124 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.crds_create }} +{{ $groupName := .Values.crds.group_name }} +{{ $groupVersion := .Values.crds.group_version }} +{{ $groupVersionFormat := printf "%s/%s" $groupName $groupVersion }} +{{ $crdName := printf "%s.%s" "daemonjobs" $groupName }} +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: {{ $crdName }} +spec: + group: {{ $groupName }} + versions: + - name: {{ $groupVersion }} + served: true + storage: true + schema: + openAPIV3Schema: + description: DaemonJob is the Schema for the daemonjobs API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: DaemonJobSpec defines the desired state of DaemonJob + properties: + selector: + description: Foo is an example field of DaemonJob. Edit DaemonJob_types.go + to remove/update + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. + The requirements are ANDed. + items: + description: A label selector requirement is a selector that contains + values, a key, and an operator that relates the key and values. + properties: + key: + description: key is the label key that the selector applies + to. + type: string + operator: + description: operator represents a key's relationship to a + set of values. Valid operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string values. If the operator + is In or NotIn, the values array must be non-empty. If the + operator is Exists or DoesNotExist, the values array must + be empty. This array is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. A single + {key,value} in the matchLabels map is equivalent to an element + of matchExpressions, whose key field is "key", the operator is + "In", and the values array contains only "value". The requirements + are ANDed. + type: object + type: object + template: + description: PodTemplateSpec describes the data a pod should have when + created from a template + properties: + metadata: + type: object + properties: + annotations: + type: object + additionalProperties: + type: string + labels: + type: object + additionalProperties: + type: string + spec: + description: 'Specification of the desired behavior of the pod. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#spec-and-status' + properties: + activeDeadlineSeconds: + description: Optional duration in seconds the pod may be active + on the node relative to StartTime before the system will actively + try to mark it failed and kill associated containers. Value + must be a positive integer. + format: int64 + type: integer + affinity: + description: If specified, the pod's scheduling constraints + properties: + nodeAffinity: + description: Describes node affinity scheduling rules for + the pod. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods + to nodes that satisfy the affinity expressions specified + by this field, but it may choose a node that violates + one or more of the expressions. The node that is most + preferred is the one with the greatest sum of weights, + i.e. for each node that meets all of the scheduling + requirements (resource request, requiredDuringScheduling + affinity expressions, etc.), compute a sum by iterating + through the elements of this field and adding "weight" + to the sum if the node matches the corresponding matchExpressions; + the node(s) with the highest sum are the most preferred. + items: + description: An empty preferred scheduling term matches + all objects with implicit weight 0 (i.e. it's a + no-op). A null preferred scheduling term matches + no objects (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated + with the corresponding weight. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: A node selector requirement + is a selector that contains values, a + key, and an operator that relates the + key and values. + properties: + key: + description: The label key that the + selector applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. + If the operator is In or NotIn, the + values array must be non-empty. If + the operator is Exists or DoesNotExist, + the values array must be empty. If + the operator is Gt or Lt, the values + array must have a single element, + which will be interpreted as an integer. + This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: A node selector requirement + is a selector that contains values, a + key, and an operator that relates the + key and values. + properties: + key: + description: The label key that the + selector applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. + If the operator is In or NotIn, the + values array must be non-empty. If + the operator is Exists or DoesNotExist, + the values array must be empty. If + the operator is Gt or Lt, the values + array must have a single element, + which will be interpreted as an integer. + This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + weight: + description: Weight associated with matching the + corresponding nodeSelectorTerm, in the range + 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the affinity requirements specified + by this field are not met at scheduling time, the + pod will not be scheduled onto the node. If the affinity + requirements specified by this field cease to be met + at some point during pod execution (e.g. due to an + update), the system may or may not try to eventually + evict the pod from its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. + The terms are ORed. + items: + description: A null or empty node selector term + matches no objects. The requirements of them + are ANDed. The TopologySelectorTerm type implements + a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: A node selector requirement + is a selector that contains values, a + key, and an operator that relates the + key and values. + properties: + key: + description: The label key that the + selector applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. + If the operator is In or NotIn, the + values array must be non-empty. If + the operator is Exists or DoesNotExist, + the values array must be empty. If + the operator is Gt or Lt, the values + array must have a single element, + which will be interpreted as an integer. + This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: A node selector requirement + is a selector that contains values, a + key, and an operator that relates the + key and values. + properties: + key: + description: The label key that the + selector applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. + If the operator is In or NotIn, the + values array must be non-empty. If + the operator is Exists or DoesNotExist, + the values array must be empty. If + the operator is Gt or Lt, the values + array must have a single element, + which will be interpreted as an integer. + This array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + type: array + required: + - nodeSelectorTerms + type: object + type: object + podAffinity: + description: Describes pod affinity scheduling rules (e.g. + co-locate this pod in the same node, zone, etc. as some + other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods + to nodes that satisfy the affinity expressions specified + by this field, but it may choose a node that violates + one or more of the expressions. The node that is most + preferred is the one with the greatest sum of weights, + i.e. for each node that meets all of the scheduling + requirements (resource request, requiredDuringScheduling + affinity expressions, etc.), compute a sum by iterating + through the elements of this field and adding "weight" + to the sum if the node has pods which matches the + corresponding podAffinityTerm; the node(s) with the + highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated + with the corresponding weight. + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: operator represents + a key's relationship to a set + of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array + of string values. If the operator + is In or NotIn, the values array + must be non-empty. If the operator + is Exists or DoesNotExist, the + values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + namespaces: + description: namespaces specifies which namespaces + the labelSelector applies to (matches against); + null or empty list means "this pod's namespace" + items: + type: string + type: array + topologyKey: + description: This pod should be co-located + (affinity) or not co-located (anti-affinity) + with the pods matching the labelSelector + in the specified namespaces, where co-located + is defined as running on a node whose value + of the label with key topologyKey matches + that of any node on which any of the selected + pods is running. Empty topologyKey is not + allowed. + type: string + required: + - topologyKey + type: object + weight: + description: weight associated with matching the + corresponding podAffinityTerm, in the range + 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the affinity requirements specified + by this field are not met at scheduling time, the + pod will not be scheduled onto the node. If the affinity + requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a + pod label update), the system may or may not try to + eventually evict the pod from its node. When there + are multiple elements, the lists of nodes corresponding + to each podAffinityTerm are intersected, i.e. all + terms must be satisfied. + items: + description: Defines a set of pods (namely those matching + the labelSelector relative to the given namespace(s)) + that this pod should be co-located (affinity) or + not co-located (anti-affinity) with, where co-located + is defined as running on a node whose value of the + label with key matches that of any + node on which a pod of the set of pods is running + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list of + label selector requirements. The requirements + are ANDed. + items: + description: A label selector requirement + is a selector that contains values, a + key, and an operator that relates the + key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only + "value". The requirements are ANDed. + type: object + type: object + namespaces: + description: namespaces specifies which namespaces + the labelSelector applies to (matches against); + null or empty list means "this pod's namespace" + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) + or not co-located (anti-affinity) with the pods + matching the labelSelector in the specified + namespaces, where co-located is defined as running + on a node whose value of the label with key + topologyKey matches that of any node on which + any of the selected pods is running. Empty topologyKey + is not allowed. + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + description: Describes pod anti-affinity scheduling rules + (e.g. avoid putting this pod in the same node, zone, etc. + as some other pod(s)). + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods + to nodes that satisfy the anti-affinity expressions + specified by this field, but it may choose a node + that violates one or more of the expressions. The + node that is most preferred is the one with the greatest + sum of weights, i.e. for each node that meets all + of the scheduling requirements (resource request, + requiredDuringScheduling anti-affinity expressions, + etc.), compute a sum by iterating through the elements + of this field and adding "weight" to the sum if the + node has pods which matches the corresponding podAffinityTerm; + the node(s) with the highest sum are the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated + with the corresponding weight. + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The + requirements are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: operator represents + a key's relationship to a set + of values. Valid operators are + In, NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array + of string values. If the operator + is In or NotIn, the values array + must be non-empty. If the operator + is Exists or DoesNotExist, the + values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + namespaces: + description: namespaces specifies which namespaces + the labelSelector applies to (matches against); + null or empty list means "this pod's namespace" + items: + type: string + type: array + topologyKey: + description: This pod should be co-located + (affinity) or not co-located (anti-affinity) + with the pods matching the labelSelector + in the specified namespaces, where co-located + is defined as running on a node whose value + of the label with key topologyKey matches + that of any node on which any of the selected + pods is running. Empty topologyKey is not + allowed. + type: string + required: + - topologyKey + type: object + weight: + description: weight associated with matching the + corresponding podAffinityTerm, in the range + 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the anti-affinity requirements specified + by this field are not met at scheduling time, the + pod will not be scheduled onto the node. If the anti-affinity + requirements specified by this field cease to be met + at some point during pod execution (e.g. due to a + pod label update), the system may or may not try to + eventually evict the pod from its node. When there + are multiple elements, the lists of nodes corresponding + to each podAffinityTerm are intersected, i.e. all + terms must be satisfied. + items: + description: Defines a set of pods (namely those matching + the labelSelector relative to the given namespace(s)) + that this pod should be co-located (affinity) or + not co-located (anti-affinity) with, where co-located + is defined as running on a node whose value of the + label with key matches that of any + node on which a pod of the set of pods is running + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list of + label selector requirements. The requirements + are ANDed. + items: + description: A label selector requirement + is a selector that contains values, a + key, and an operator that relates the + key and values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only + "value". The requirements are ANDed. + type: object + type: object + namespaces: + description: namespaces specifies which namespaces + the labelSelector applies to (matches against); + null or empty list means "this pod's namespace" + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) + or not co-located (anti-affinity) with the pods + matching the labelSelector in the specified + namespaces, where co-located is defined as running + on a node whose value of the label with key + topologyKey matches that of any node on which + any of the selected pods is running. Empty topologyKey + is not allowed. + type: string + required: + - topologyKey + type: object + type: array + type: object + type: object + automountServiceAccountToken: + description: AutomountServiceAccountToken indicates whether + a service account token should be automatically mounted. + type: boolean + containers: + description: List of containers belonging to the pod. Containers + cannot currently be added or removed. There must be at least + one container in a Pod. Cannot be updated. + items: + description: A single application container that you want + to run within a pod. + properties: + args: + description: 'Arguments to the entrypoint. The docker + image''s CMD is used if this is not provided. Variable + references $(VAR_NAME) are expanded using the container''s + environment. If a variable cannot be resolved, the reference + in the input string will be unchanged. The $(VAR_NAME) + syntax can be escaped with a double $$, ie: $$(VAR_NAME). + Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + items: + type: string + type: array + command: + description: 'Entrypoint array. Not executed within a + shell. The docker image''s ENTRYPOINT is used if this + is not provided. Variable references $(VAR_NAME) are + expanded using the container''s environment. If a variable + cannot be resolved, the reference in the input string + will be unchanged. The $(VAR_NAME) syntax can be escaped + with a double $$, ie: $$(VAR_NAME). Escaped references + will never be expanded, regardless of whether the variable + exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + items: + type: string + type: array + env: + description: List of environment variables to set in the + container. Cannot be updated. + items: + description: EnvVar represents an environment variable + present in a Container. + properties: + name: + description: Name of the environment variable. Must + be a C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) are + expanded using the previous defined environment + variables in the container and any service environment + variables. If a variable cannot be resolved, the + reference in the input string will be unchanged. + The $(VAR_NAME) syntax can be escaped with a double + $$, ie: $$(VAR_NAME). Escaped references will + never be expanded, regardless of whether the variable + exists or not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment variable's + value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap + or its key must be defined + type: boolean + required: + - key + type: object + fieldRef: + description: 'Selects a field of the pod: supports + metadata.name, metadata.namespace, metadata.labels, + metadata.annotations, spec.nodeName, spec.serviceAccountName, + status.hostIP, status.podIP, status.podIPs.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select + in the specified API version. + type: string + required: + - fieldPath + type: object + resourceFieldRef: + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, limits.ephemeral-storage, requests.cpu, + requests.memory and requests.ephemeral-storage) + are currently supported.' + properties: + containerName: + description: 'Container name: required for + volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format + of the exposed resources, defaults to + "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + secretKeyRef: + description: Selects a key of a secret in the + pod's namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret + or its key must be defined + type: boolean + required: + - key + type: object + type: object + required: + - name + type: object + type: array + envFrom: + description: List of sources to populate environment variables + in the container. The keys defined within a source must + be a C_IDENTIFIER. All invalid keys will be reported + as an event when the container is starting. When a key + exists in multiple sources, the value associated with + the last source will take precedence. Values defined + by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source of + a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap must + be defined + type: boolean + type: object + prefix: + description: An optional identifier to prepend to + each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret must + be defined + type: boolean + type: object + type: object + type: array + image: + description: 'Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images + This field is optional to allow higher level config + management to default or override container images in + workload controllers like Deployments and StatefulSets.' + type: string + imagePullPolicy: + description: 'Image pull policy. One of Always, Never, + IfNotPresent. Defaults to Always if :latest tag is specified, + or IfNotPresent otherwise. Cannot be updated. More info: + https://kubernetes.io/docs/concepts/containers/images#updating-images' + type: string + lifecycle: + description: Actions that the management system should + take in response to container lifecycle events. Cannot + be updated. + properties: + postStart: + description: 'PostStart is called immediately after + a container is created. If the handler fails, the + container is terminated and restarted according + to its restart policy. Other management of the container + blocks until the hook completes. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + properties: + exec: + description: One and only one of the following + should be specified. Exec specifies the action + to take. + properties: + command: + description: Command is the command line to + execute inside the container, the working + directory for the command is root ('/') + in the container's filesystem. The command + is simply exec'd, it is not run inside a + shell, so traditional shell instructions + ('|', etc) won't work. To use a shell, you + need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy + and non-zero is unhealthy. + items: + type: string + type: array + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: Host name to connect to, defaults + to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to + access on the container. Number must be + in the range 1 to 65535. Name must be an + IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting + to the host. Defaults to HTTP. + type: string + required: + - port + type: object + tcpSocket: + description: 'TCPSocket specifies an action involving + a TCP port. TCP hooks not yet supported TODO: + implement a realistic TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to + access on the container. Number must be + in the range 1 to 65535. Name must be an + IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: 'PreStop is called immediately before + a container is terminated due to an API request + or management event such as liveness/startup probe + failure, preemption, resource contention, etc. The + handler is not called if the container crashes or + exits. The reason for termination is passed to the + handler. The Pod''s termination grace period countdown + begins before the PreStop hooked is executed. Regardless + of the outcome of the handler, the container will + eventually terminate within the Pod''s termination + grace period. Other management of the container + blocks until the hook completes or until the termination + grace period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + properties: + exec: + description: One and only one of the following + should be specified. Exec specifies the action + to take. + properties: + command: + description: Command is the command line to + execute inside the container, the working + directory for the command is root ('/') + in the container's filesystem. The command + is simply exec'd, it is not run inside a + shell, so traditional shell instructions + ('|', etc) won't work. To use a shell, you + need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy + and non-zero is unhealthy. + items: + type: string + type: array + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: Host name to connect to, defaults + to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to + access on the container. Number must be + in the range 1 to 65535. Name must be an + IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting + to the host. Defaults to HTTP. + type: string + required: + - port + type: object + tcpSocket: + description: 'TCPSocket specifies an action involving + a TCP port. TCP hooks not yet supported TODO: + implement a realistic TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to + access on the container. Number must be + in the range 1 to 65535. Name must be an + IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + name: + description: Name of the container specified as a DNS_LABEL. + Each container in a pod must have a unique name (DNS_LABEL). + Cannot be updated. + type: string + ports: + description: List of ports to expose from the container. + Exposing a port here gives the system additional information + about the network connections a container uses, but + is primarily informational. Not specifying a port here + DOES NOT prevent that port from being exposed. Any port + which is listening on the default "0.0.0.0" address + inside a container will be accessible from the network. + Cannot be updated. + items: + description: ContainerPort represents a network port + in a single container. + properties: + containerPort: + description: Number of port to expose on the pod's + IP address. This must be a valid port number, + 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external port + to. + type: string + hostPort: + description: Number of port to expose on the host. + If specified, this must be a valid port number, + 0 < x < 65536. If HostNetwork is specified, this + must match ContainerPort. Most containers do not + need this. + format: int32 + type: integer + name: + description: If specified, this must be an IANA_SVC_NAME + and unique within the pod. Each named port in + a pod must have a unique name. Name for the port + that can be referred to by services. + type: string + protocol: + description: Protocol for port. Must be UDP, TCP, + or SCTP. Defaults to "TCP". + type: string + default: "TCP" + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + resources: + description: 'Compute Resources required by this container. + Cannot be updated. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount + of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount + of compute resources required. If Requests is omitted + for a container, it defaults to Limits if that is + explicitly specified, otherwise to an implementation-defined + value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + type: object + securityContext: + description: 'Security options the pod should run with. + More info: https://kubernetes.io/docs/concepts/policy/security-context/ + More info: https://kubernetes.io/docs/tasks/configure-pod-container/security-context/' + properties: + allowPrivilegeEscalation: + description: 'AllowPrivilegeEscalation controls whether + a process can gain more privileges than its parent + process. This bool directly controls if the no_new_privs + flag will be set on the container process. AllowPrivilegeEscalation + is true always when the container is: 1) run as + Privileged 2) has CAP_SYS_ADMIN' + type: boolean + capabilities: + description: The capabilities to add/drop when running + containers. Defaults to the default set of capabilities + granted by the container runtime. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + type: object + privileged: + description: Run container in privileged mode. Processes + in privileged containers are essentially equivalent + to root on the host. Defaults to false. + type: boolean + procMount: + description: procMount denotes the type of proc mount + to use for the containers. The default is DefaultProcMount + which uses the container runtime defaults for readonly + paths and masked paths. This requires the ProcMountType + feature flag to be enabled. + type: string + readOnlyRootFilesystem: + description: Whether this container has a read-only + root filesystem. Default is false. + type: boolean + runAsGroup: + description: The GID to run the entrypoint of the + container process. Uses runtime default if unset. + May also be set in PodSecurityContext. If set in + both SecurityContext and PodSecurityContext, the + value specified in SecurityContext takes precedence. + format: int64 + type: integer + runAsNonRoot: + description: Indicates that the container must run + as a non-root user. If true, the Kubelet will validate + the image at runtime to ensure that it does not + run as UID 0 (root) and fail to start the container + if it does. If unset or false, no such validation + will be performed. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: The UID to run the entrypoint of the + container process. Defaults to user specified in + image metadata if unspecified. May also be set in + PodSecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified in SecurityContext + takes precedence. + format: int64 + type: integer + seLinuxOptions: + description: The SELinux context to be applied to + the container. If unspecified, the container runtime + will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence. + properties: + level: + description: Level is SELinux level label that + applies to the container. + type: string + role: + description: Role is a SELinux role label that + applies to the container. + type: string + type: + description: Type is a SELinux type label that + applies to the container. + type: string + user: + description: User is a SELinux user label that + applies to the container. + type: string + type: object + windowsOptions: + description: The Windows specific settings applied + to all containers. If unspecified, the options from + the PodSecurityContext will be used. If set in both + SecurityContext and PodSecurityContext, the value + specified in SecurityContext takes precedence. + properties: + gmsaCredentialSpec: + description: GMSACredentialSpec is where the GMSA + admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) + inlines the contents of the GMSA credential + spec named by the GMSACredentialSpecName field. + This field is alpha-level and is only honored + by servers that enable the WindowsGMSA feature + flag. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name + of the GMSA credential spec to use. This field + is alpha-level and is only honored by servers + that enable the WindowsGMSA feature flag. + type: string + runAsUserName: + description: The UserName in Windows to run the + entrypoint of the container process. Defaults + to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set + in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes + precedence. This field is beta-level and may + be disabled with the WindowsRunAsUserName feature + flag. + type: string + type: object + type: object + stdin: + description: Whether this container should allocate a + buffer for stdin in the container runtime. If this is + not set, reads from stdin in the container will always + result in EOF. Default is false. + type: boolean + stdinOnce: + description: Whether the container runtime should close + the stdin channel after it has been opened by a single + attach. When stdin is true the stdin stream will remain + open across multiple attach sessions. If stdinOnce is + set to true, stdin is opened on container start, is + empty until the first client attaches to stdin, and + then remains open and accepts data until the client + disconnects, at which time stdin is closed and remains + closed until the container is restarted. If this flag + is false, a container processes that reads from stdin + will never receive an EOF. Default is false + type: boolean + terminationMessagePath: + description: 'Optional: Path at which the file to which + the container''s termination message will be written + is mounted into the container''s filesystem. Message + written is intended to be brief final status, such as + an assertion failure message. Will be truncated by the + node if greater than 4096 bytes. The total message length + across all containers will be limited to 12kb. Defaults + to /dev/termination-log. Cannot be updated.' + type: string + terminationMessagePolicy: + description: Indicate how the termination message should + be populated. File will use the contents of terminationMessagePath + to populate the container status message on both success + and failure. FallbackToLogsOnError will use the last + chunk of container log output if the termination message + file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, + whichever is smaller. Defaults to File. Cannot be updated. + type: string + tty: + description: Whether this container should allocate a + TTY for itself, also requires 'stdin' to be true. Default + is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block devices + to be used by the container. This is a beta feature. + items: + description: volumeDevice describes a mapping of a raw + block device within a container. + properties: + devicePath: + description: devicePath is the path inside of the + container that the device will be mapped to. + type: string + name: + description: name must match the name of a persistentVolumeClaim + in the pod + type: string + required: + - devicePath + - name + type: object + type: array + volumeMounts: + description: Pod volumes to mount into the container's + filesystem. Cannot be updated. + items: + description: VolumeMount describes a mounting of a Volume + within a container. + properties: + mountPath: + description: Path within the container at which + the volume should be mounted. Must not contain + ':'. + type: string + mountPropagation: + description: mountPropagation determines how mounts + are propagated from the host to container and + the other way around. When not set, MountPropagationNone + is used. This field is beta in 1.10. + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: Mounted read-only if true, read-write + otherwise (false or unspecified). Defaults to + false. + type: boolean + subPath: + description: Path within the volume from which the + container's volume should be mounted. Defaults + to "" (volume's root). + type: string + subPathExpr: + description: Expanded path within the volume from + which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable + references $(VAR_NAME) are expanded using the + container's environment. Defaults to "" (volume's + root). SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + workingDir: + description: Container's working directory. If not specified, + the container runtime's default will be used, which + might be configured in the container image. Cannot be + updated. + type: string + required: + - name + type: object + type: array + dnsConfig: + description: Specifies the DNS parameters of a pod. Parameters + specified here will be merged to the generated DNS configuration + based on DNSPolicy. + properties: + nameservers: + description: A list of DNS name server IP addresses. This + will be appended to the base nameservers generated from + DNSPolicy. Duplicated nameservers will be removed. + items: + type: string + type: array + options: + description: A list of DNS resolver options. This will be + merged with the base options generated from DNSPolicy. + Duplicated entries will be removed. Resolution options + given in Options will override those that appear in the + base DNSPolicy. + items: + description: PodDNSConfigOption defines DNS resolver options + of a pod. + properties: + name: + description: Required. + type: string + value: + type: string + type: object + type: array + searches: + description: A list of DNS search domains for host-name + lookup. This will be appended to the base search paths + generated from DNSPolicy. Duplicated search paths will + be removed. + items: + type: string + type: array + type: object + dnsPolicy: + description: Set DNS policy for the pod. Defaults to "ClusterFirst". + Valid values are 'ClusterFirstWithHostNet', 'ClusterFirst', + 'Default' or 'None'. DNS parameters given in DNSConfig will + be merged with the policy selected with DNSPolicy. To have + DNS options set along with hostNetwork, you have to specify + DNS policy explicitly to 'ClusterFirstWithHostNet'. + type: string + enableServiceLinks: + description: 'EnableServiceLinks indicates whether information + about services should be injected into pod''s environment + variables, matching the syntax of Docker links. Optional: + Defaults to true.' + type: boolean + ephemeralContainers: + description: List of ephemeral containers run in this pod. Ephemeral + containers may be run in an existing pod to perform user-initiated + actions such as debugging. This list cannot be specified when + creating a pod, and it cannot be modified by updating the + pod spec. In order to add an ephemeral container to an existing + pod, use the pod's ephemeralcontainers subresource. This field + is alpha-level and is only honored by servers that enable + the EphemeralContainers feature. + items: + description: An EphemeralContainer is a container that may + be added temporarily to an existing pod for user-initiated + activities such as debugging. Ephemeral containers have + no resource or scheduling guarantees, and they will not + be restarted when they exit or when a pod is removed or + restarted. If an ephemeral container causes a pod to exceed + its resource allocation, the pod may be evicted. Ephemeral + containers may not be added by directly updating the pod + spec. They must be added via the pod's ephemeralcontainers + subresource, and they will appear in the pod spec once added. + This is an alpha feature enabled by the EphemeralContainers + feature flag. + properties: + args: + description: 'Arguments to the entrypoint. The docker + image''s CMD is used if this is not provided. Variable + references $(VAR_NAME) are expanded using the container''s + environment. If a variable cannot be resolved, the reference + in the input string will be unchanged. The $(VAR_NAME) + syntax can be escaped with a double $$, ie: $$(VAR_NAME). + Escaped references will never be expanded, regardless + of whether the variable exists or not. Cannot be updated. + More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + items: + type: string + type: array + command: + description: 'Entrypoint array. Not executed within a + shell. The docker image''s ENTRYPOINT is used if this + is not provided. Variable references $(VAR_NAME) are + expanded using the container''s environment. If a variable + cannot be resolved, the reference in the input string + will be unchanged. The $(VAR_NAME) syntax can be escaped + with a double $$, ie: $$(VAR_NAME). Escaped references + will never be expanded, regardless of whether the variable + exists or not. Cannot be updated. More info: https://kubernetes.io/docs/tasks/inject-data-application/define-command-argument-container/#running-a-command-in-a-shell' + items: + type: string + type: array + env: + description: List of environment variables to set in the + container. Cannot be updated. + items: + description: EnvVar represents an environment variable + present in a Container. + properties: + name: + description: Name of the environment variable. Must + be a C_IDENTIFIER. + type: string + value: + description: 'Variable references $(VAR_NAME) are + expanded using the previous defined environment + variables in the container and any service environment + variables. If a variable cannot be resolved, the + reference in the input string will be unchanged. + The $(VAR_NAME) syntax can be escaped with a double + $$, ie: $$(VAR_NAME). Escaped references will + never be expanded, regardless of whether the variable + exists or not. Defaults to "".' + type: string + valueFrom: + description: Source for the environment variable's + value. Cannot be used if value is not empty. + properties: + configMapKeyRef: + description: Selects a key of a ConfigMap. + properties: + key: + description: The key to select. + type: string + name: + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap + or its key must be defined + type: boolean + required: + - key + type: object + fieldRef: + description: 'Selects a field of the pod: supports + metadata.name, metadata.namespace, metadata.labels, + metadata.annotations, spec.nodeName, spec.serviceAccountName, + status.hostIP, status.podIP, status.podIPs.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select + in the specified API version. + type: string + required: + - fieldPath + type: object + resourceFieldRef: + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, limits.ephemeral-storage, requests.cpu, + requests.memory and requests.ephemeral-storage) + are currently supported.' + properties: + containerName: + description: 'Container name: required for + volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format + of the exposed resources, defaults to + "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + secretKeyRef: + description: Selects a key of a secret in the + pod's namespace + properties: + key: + description: The key of the secret to select + from. Must be a valid secret key. + type: string + name: + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret + or its key must be defined + type: boolean + required: + - key + type: object + type: object + required: + - name + type: object + type: array + envFrom: + description: List of sources to populate environment variables + in the container. The keys defined within a source must + be a C_IDENTIFIER. All invalid keys will be reported + as an event when the container is starting. When a key + exists in multiple sources, the value associated with + the last source will take precedence. Values defined + by an Env with a duplicate key will take precedence. + Cannot be updated. + items: + description: EnvFromSource represents the source of + a set of ConfigMaps + properties: + configMapRef: + description: The ConfigMap to select from + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap must + be defined + type: boolean + type: object + prefix: + description: An optional identifier to prepend to + each key in the ConfigMap. Must be a C_IDENTIFIER. + type: string + secretRef: + description: The Secret to select from + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret must + be defined + type: boolean + type: object + type: object + type: array + image: + description: 'Docker image name. More info: https://kubernetes.io/docs/concepts/containers/images' + type: string + imagePullPolicy: + description: 'Image pull policy. One of Always, Never, + IfNotPresent. Defaults to Always if :latest tag is specified, + or IfNotPresent otherwise. Cannot be updated. More info: + https://kubernetes.io/docs/concepts/containers/images#updating-images' + type: string + lifecycle: + description: Lifecycle is not allowed for ephemeral containers. + properties: + postStart: + description: 'PostStart is called immediately after + a container is created. If the handler fails, the + container is terminated and restarted according + to its restart policy. Other management of the container + blocks until the hook completes. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + properties: + exec: + description: One and only one of the following + should be specified. Exec specifies the action + to take. + properties: + command: + description: Command is the command line to + execute inside the container, the working + directory for the command is root ('/') + in the container's filesystem. The command + is simply exec'd, it is not run inside a + shell, so traditional shell instructions + ('|', etc) won't work. To use a shell, you + need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy + and non-zero is unhealthy. + items: + type: string + type: array + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: Host name to connect to, defaults + to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to + access on the container. Number must be + in the range 1 to 65535. Name must be an + IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting + to the host. Defaults to HTTP. + type: string + required: + - port + type: object + tcpSocket: + description: 'TCPSocket specifies an action involving + a TCP port. TCP hooks not yet supported TODO: + implement a realistic TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to + access on the container. Number must be + in the range 1 to 65535. Name must be an + IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + description: 'PreStop is called immediately before + a container is terminated due to an API request + or management event such as liveness/startup probe + failure, preemption, resource contention, etc. The + handler is not called if the container crashes or + exits. The reason for termination is passed to the + handler. The Pod''s termination grace period countdown + begins before the PreStop hooked is executed. Regardless + of the outcome of the handler, the container will + eventually terminate within the Pod''s termination + grace period. Other management of the container + blocks until the hook completes or until the termination + grace period is reached. More info: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/#container-hooks' + properties: + exec: + description: One and only one of the following + should be specified. Exec specifies the action + to take. + properties: + command: + description: Command is the command line to + execute inside the container, the working + directory for the command is root ('/') + in the container's filesystem. The command + is simply exec'd, it is not run inside a + shell, so traditional shell instructions + ('|', etc) won't work. To use a shell, you + need to explicitly call out to that shell. + Exit status of 0 is treated as live/healthy + and non-zero is unhealthy. + items: + type: string + type: array + type: object + httpGet: + description: HTTPGet specifies the http request + to perform. + properties: + host: + description: Host name to connect to, defaults + to the pod IP. You probably want to set + "Host" in httpHeaders instead. + type: string + httpHeaders: + description: Custom headers to set in the + request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom + header to be used in HTTP probes + properties: + name: + description: The header field name + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to + access on the container. Number must be + in the range 1 to 65535. Name must be an + IANA_SVC_NAME. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting + to the host. Defaults to HTTP. + type: string + required: + - port + type: object + tcpSocket: + description: 'TCPSocket specifies an action involving + a TCP port. TCP hooks not yet supported TODO: + implement a realistic TCP lifecycle hook' + properties: + host: + description: 'Optional: Host name to connect + to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to + access on the container. Number must be + in the range 1 to 65535. Name must be an + IANA_SVC_NAME. + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + type: object + name: + description: Name of the ephemeral container specified + as a DNS_LABEL. This name must be unique among all containers, + init containers and ephemeral containers. + type: string + ports: + description: Ports are not allowed for ephemeral containers. + items: + description: ContainerPort represents a network port + in a single container. + properties: + containerPort: + description: Number of port to expose on the pod's + IP address. This must be a valid port number, + 0 < x < 65536. + format: int32 + type: integer + hostIP: + description: What host IP to bind the external port + to. + type: string + hostPort: + description: Number of port to expose on the host. + If specified, this must be a valid port number, + 0 < x < 65536. If HostNetwork is specified, this + must match ContainerPort. Most containers do not + need this. + format: int32 + type: integer + name: + description: If specified, this must be an IANA_SVC_NAME + and unique within the pod. Each named port in + a pod must have a unique name. Name for the port + that can be referred to by services. + type: string + protocol: + description: Protocol for port. Must be UDP, TCP, + or SCTP. Defaults to "TCP". + type: string + required: + - containerPort + type: object + type: array + resources: + description: Resources are not allowed for ephemeral containers. + Ephemeral containers use spare resources already allocated + to the pod. + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount + of compute resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount + of compute resources required. If Requests is omitted + for a container, it defaults to Limits if that is + explicitly specified, otherwise to an implementation-defined + value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' + type: object + type: object + securityContext: + description: SecurityContext is not allowed for ephemeral + containers. + properties: + allowPrivilegeEscalation: + description: 'AllowPrivilegeEscalation controls whether + a process can gain more privileges than its parent + process. This bool directly controls if the no_new_privs + flag will be set on the container process. AllowPrivilegeEscalation + is true always when the container is: 1) run as + Privileged 2) has CAP_SYS_ADMIN' + type: boolean + capabilities: + description: The capabilities to add/drop when running + containers. Defaults to the default set of capabilities + granted by the container runtime. + properties: + add: + description: Added capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + drop: + description: Removed capabilities + items: + description: Capability represent POSIX capabilities + type + type: string + type: array + type: object + privileged: + description: Run container in privileged mode. Processes + in privileged containers are essentially equivalent + to root on the host. Defaults to false. + type: boolean + procMount: + description: procMount denotes the type of proc mount + to use for the containers. The default is DefaultProcMount + which uses the container runtime defaults for readonly + paths and masked paths. This requires the ProcMountType + feature flag to be enabled. + type: string + readOnlyRootFilesystem: + description: Whether this container has a read-only + root filesystem. Default is false. + type: boolean + runAsGroup: + description: The GID to run the entrypoint of the + container process. Uses runtime default if unset. + May also be set in PodSecurityContext. If set in + both SecurityContext and PodSecurityContext, the + value specified in SecurityContext takes precedence. + format: int64 + type: integer + runAsNonRoot: + description: Indicates that the container must run + as a non-root user. If true, the Kubelet will validate + the image at runtime to ensure that it does not + run as UID 0 (root) and fail to start the container + if it does. If unset or false, no such validation + will be performed. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence. + type: boolean + runAsUser: + description: The UID to run the entrypoint of the + container process. Defaults to user specified in + image metadata if unspecified. May also be set in + PodSecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified in SecurityContext + takes precedence. + format: int64 + type: integer + seLinuxOptions: + description: The SELinux context to be applied to + the container. If unspecified, the container runtime + will allocate a random SELinux context for each + container. May also be set in PodSecurityContext. If + set in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes precedence. + properties: + level: + description: Level is SELinux level label that + applies to the container. + type: string + role: + description: Role is a SELinux role label that + applies to the container. + type: string + type: + description: Type is a SELinux type label that + applies to the container. + type: string + user: + description: User is a SELinux user label that + applies to the container. + type: string + type: object + windowsOptions: + description: The Windows specific settings applied + to all containers. If unspecified, the options from + the PodSecurityContext will be used. If set in both + SecurityContext and PodSecurityContext, the value + specified in SecurityContext takes precedence. + properties: + gmsaCredentialSpec: + description: GMSACredentialSpec is where the GMSA + admission webhook (https://github.com/kubernetes-sigs/windows-gmsa) + inlines the contents of the GMSA credential + spec named by the GMSACredentialSpecName field. + This field is alpha-level and is only honored + by servers that enable the WindowsGMSA feature + flag. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name + of the GMSA credential spec to use. This field + is alpha-level and is only honored by servers + that enable the WindowsGMSA feature flag. + type: string + runAsUserName: + description: The UserName in Windows to run the + entrypoint of the container process. Defaults + to the user specified in image metadata if unspecified. + May also be set in PodSecurityContext. If set + in both SecurityContext and PodSecurityContext, + the value specified in SecurityContext takes + precedence. This field is beta-level and may + be disabled with the WindowsRunAsUserName feature + flag. + type: string + type: object + type: object + stdin: + description: Whether this container should allocate a + buffer for stdin in the container runtime. If this is + not set, reads from stdin in the container will always + result in EOF. Default is false. + type: boolean + stdinOnce: + description: Whether the container runtime should close + the stdin channel after it has been opened by a single + attach. When stdin is true the stdin stream will remain + open across multiple attach sessions. If stdinOnce is + set to true, stdin is opened on container start, is + empty until the first client attaches to stdin, and + then remains open and accepts data until the client + disconnects, at which time stdin is closed and remains + closed until the container is restarted. If this flag + is false, a container processes that reads from stdin + will never receive an EOF. Default is false + type: boolean + targetContainerName: + description: If set, the name of the container from PodSpec + that this ephemeral container targets. The ephemeral + container will be run in the namespaces (IPC, PID, etc) + of this container. If not set then the ephemeral container + is run in whatever namespaces are shared for the pod. + Note that the container runtime must support this feature. + type: string + terminationMessagePath: + description: 'Optional: Path at which the file to which + the container''s termination message will be written + is mounted into the container''s filesystem. Message + written is intended to be brief final status, such as + an assertion failure message. Will be truncated by the + node if greater than 4096 bytes. The total message length + across all containers will be limited to 12kb. Defaults + to /dev/termination-log. Cannot be updated.' + type: string + terminationMessagePolicy: + description: Indicate how the termination message should + be populated. File will use the contents of terminationMessagePath + to populate the container status message on both success + and failure. FallbackToLogsOnError will use the last + chunk of container log output if the termination message + file is empty and the container exited with an error. + The log output is limited to 2048 bytes or 80 lines, + whichever is smaller. Defaults to File. Cannot be updated. + type: string + tty: + description: Whether this container should allocate a + TTY for itself, also requires 'stdin' to be true. Default + is false. + type: boolean + volumeDevices: + description: volumeDevices is the list of block devices + to be used by the container. This is a beta feature. + items: + description: volumeDevice describes a mapping of a raw + block device within a container. + properties: + devicePath: + description: devicePath is the path inside of the + container that the device will be mapped to. + type: string + name: + description: name must match the name of a persistentVolumeClaim + in the pod + type: string + required: + - devicePath + - name + type: object + type: array + volumeMounts: + description: Pod volumes to mount into the container's + filesystem. Cannot be updated. + items: + description: VolumeMount describes a mounting of a Volume + within a container. + properties: + mountPath: + description: Path within the container at which + the volume should be mounted. Must not contain + ':'. + type: string + mountPropagation: + description: mountPropagation determines how mounts + are propagated from the host to container and + the other way around. When not set, MountPropagationNone + is used. This field is beta in 1.10. + type: string + name: + description: This must match the Name of a Volume. + type: string + readOnly: + description: Mounted read-only if true, read-write + otherwise (false or unspecified). Defaults to + false. + type: boolean + subPath: + description: Path within the volume from which the + container's volume should be mounted. Defaults + to "" (volume's root). + type: string + subPathExpr: + description: Expanded path within the volume from + which the container's volume should be mounted. + Behaves similarly to SubPath but environment variable + references $(VAR_NAME) are expanded using the + container's environment. Defaults to "" (volume's + root). SubPathExpr and SubPath are mutually exclusive. + type: string + required: + - mountPath + - name + type: object + type: array + workingDir: + description: Container's working directory. If not specified, + the container runtime's default will be used, which + might be configured in the container image. Cannot be + updated. + type: string + required: + - name + type: object + type: array + hostAliases: + description: HostAliases is an optional list of hosts and IPs + that will be injected into the pod's hosts file if specified. + This is only valid for non-hostNetwork pods. + items: + description: HostAlias holds the mapping between IP and hostnames + that will be injected as an entry in the pod's hosts file. + properties: + hostnames: + description: Hostnames for the above IP address. + items: + type: string + type: array + ip: + description: IP address of the host file entry. + type: string + type: object + type: array + hostIPC: + description: 'Use the host''s ipc namespace. Optional: Default + to false.' + type: boolean + hostNetwork: + description: Host networking requested for this pod. Use the + host's network namespace. If this option is set, the ports + that will be used must be specified. Default to false. + type: boolean + hostPID: + description: 'Use the host''s pid namespace. Optional: Default + to false.' + type: boolean + hostname: + description: Specifies the hostname of the Pod If not specified, + the pod's hostname will be set to a system-defined value. + type: string + imagePullSecrets: + description: 'ImagePullSecrets is an optional list of references + to secrets in the same namespace to use for pulling any of + the images used by this PodSpec. If specified, these secrets + will be passed to individual puller implementations for them + to use. For example, in the case of docker, only DockerConfig + type secrets are honored. More info: https://kubernetes.io/docs/concepts/containers/images#specifying-imagepullsecrets-on-a-pod' + items: + description: LocalObjectReference contains enough information + to let you locate the referenced object inside the same + namespace. + properties: + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, uid?' + type: string + type: object + type: array + nodeName: + description: NodeName is a request to schedule this pod onto + a specific node. If it is non-empty, the scheduler simply + schedules this pod onto that node, assuming that it fits resource + requirements. + type: string + nodeSelector: + additionalProperties: + type: string + description: 'NodeSelector is a selector which must be true + for the pod to fit on a node. Selector which must match a + node''s labels for the pod to be scheduled on that node. More + info: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/' + type: object + overhead: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Overhead represents the resource overhead associated + with running a pod for a given RuntimeClass. This field will + be autopopulated at admission time by the RuntimeClass admission + controller. If the RuntimeClass admission controller is enabled, + overhead must not be set in Pod create requests. The RuntimeClass + admission controller will reject Pod create requests which + have the overhead already set. If RuntimeClass is configured + and selected in the PodSpec, Overhead will be set to the value + defined in the corresponding RuntimeClass, otherwise it will + remain unset and treated as zero. More info: https://git.k8s.io/enhancements/keps/sig-node/20190226-pod-overhead.md + This field is alpha-level as of Kubernetes v1.16, and is only + honored by servers that enable the PodOverhead feature.' + type: object + preemptionPolicy: + description: PreemptionPolicy is the Policy for preempting pods + with lower priority. One of Never, PreemptLowerPriority. Defaults + to PreemptLowerPriority if unset. This field is alpha-level + and is only honored by servers that enable the NonPreemptingPriority + feature. + type: string + priority: + description: The priority value. Various system components use + this field to find the priority of the pod. When Priority + Admission Controller is enabled, it prevents users from setting + this field. The admission controller populates this field + from PriorityClassName. The higher the value, the higher the + priority. + format: int32 + type: integer + priorityClassName: + description: If specified, indicates the pod's priority. "system-node-critical" + and "system-cluster-critical" are two special keywords which + indicate the highest priorities with the former being the + highest priority. Any other name must be defined by creating + a PriorityClass object with that name. If not specified, the + pod priority will be default or zero if there is no default. + type: string + restartPolicy: + description: 'Restart policy for all containers within the pod. + One of Always, OnFailure, Never. Default to Always. More info: + https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#restart-policy' + type: string + runtimeClassName: + description: 'RuntimeClassName refers to a RuntimeClass object + in the node.k8s.io group, which should be used to run this + pod. If no RuntimeClass resource matches the named class, + the pod will not be run. If unset or empty, the "legacy" RuntimeClass + will be used, which is an implicit class with an empty definition + that uses the default runtime handler. More info: https://git.k8s.io/enhancements/keps/sig-node/runtime-class.md + This is a beta feature as of Kubernetes v1.14.' + type: string + schedulerName: + description: If specified, the pod will be dispatched by specified + scheduler. If not specified, the pod will be dispatched by + default scheduler. + type: string + securityContext: + description: 'SecurityContext holds pod-level security attributes + and common container settings. Optional: Defaults to empty. See + type description for default values of each field.' + properties: + fsGroup: + description: "A special supplemental group that applies + to all containers in a pod. Some volume types allow the + Kubelet to change the ownership of that volume to be owned + by the pod: \n 1. The owning GID will be the FSGroup 2. + The setgid bit is set (new files created in the volume + will be owned by FSGroup) 3. The permission bits are OR'd + with rw-rw---- \n If unset, the Kubelet will not modify + the ownership and permissions of any volume." + format: int64 + type: integer + runAsGroup: + description: The GID to run the entrypoint of the container + process. Uses runtime default if unset. May also be set + in SecurityContext. If set in both SecurityContext and + PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. + format: int64 + type: integer + runAsNonRoot: + description: Indicates that the container must run as a + non-root user. If true, the Kubelet will validate the + image at runtime to ensure that it does not run as UID + 0 (root) and fail to start the container if it does. If + unset or false, no such validation will be performed. + May also be set in SecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified in SecurityContext + takes precedence. + type: boolean + runAsUser: + description: The UID to run the entrypoint of the container + process. Defaults to user specified in image metadata + if unspecified. May also be set in SecurityContext. If + set in both SecurityContext and PodSecurityContext, the + value specified in SecurityContext takes precedence for + that container. + format: int64 + type: integer + seLinuxOptions: + description: The SELinux context to be applied to all containers. + If unspecified, the container runtime will allocate a + random SELinux context for each container. May also be + set in SecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified in SecurityContext + takes precedence for that container. + properties: + level: + description: Level is SELinux level label that applies + to the container. + type: string + role: + description: Role is a SELinux role label that applies + to the container. + type: string + type: + description: Type is a SELinux type label that applies + to the container. + type: string + user: + description: User is a SELinux user label that applies + to the container. + type: string + type: object + supplementalGroups: + description: A list of groups applied to the first process + run in each container, in addition to the container's + primary GID. If unspecified, no groups will be added + to any container. + items: + format: int64 + type: integer + type: array + sysctls: + description: Sysctls hold a list of namespaced sysctls used + for the pod. Pods with unsupported sysctls (by the container + runtime) might fail to launch. + items: + description: Sysctl defines a kernel parameter to be set + properties: + name: + description: Name of a property to set + type: string + value: + description: Value of a property to set + type: string + required: + - name + - value + type: object + type: array + windowsOptions: + description: The Windows specific settings applied to all + containers. If unspecified, the options within a container's + SecurityContext will be used. If set in both SecurityContext + and PodSecurityContext, the value specified in SecurityContext + takes precedence. + properties: + gmsaCredentialSpec: + description: GMSACredentialSpec is where the GMSA admission + webhook (https://github.com/kubernetes-sigs/windows-gmsa) + inlines the contents of the GMSA credential spec named + by the GMSACredentialSpecName field. This field is + alpha-level and is only honored by servers that enable + the WindowsGMSA feature flag. + type: string + gmsaCredentialSpecName: + description: GMSACredentialSpecName is the name of the + GMSA credential spec to use. This field is alpha-level + and is only honored by servers that enable the WindowsGMSA + feature flag. + type: string + runAsUserName: + description: The UserName in Windows to run the entrypoint + of the container process. Defaults to the user specified + in image metadata if unspecified. May also be set + in PodSecurityContext. If set in both SecurityContext + and PodSecurityContext, the value specified in SecurityContext + takes precedence. This field is beta-level and may + be disabled with the WindowsRunAsUserName feature + flag. + type: string + type: object + type: object + serviceAccount: + description: 'DeprecatedServiceAccount is a depreciated alias + for ServiceAccountName. Deprecated: Use serviceAccountName + instead.' + type: string + serviceAccountName: + description: 'ServiceAccountName is the name of the ServiceAccount + to use to run this pod. More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/' + type: string + shareProcessNamespace: + description: 'Share a single process namespace between all of + the containers in a pod. When this is set containers will + be able to view and signal processes from other containers + in the same pod, and the first process in each container will + not be assigned PID 1. HostPID and ShareProcessNamespace cannot + both be set. Optional: Default to false.' + type: boolean + subdomain: + description: If specified, the fully qualified Pod hostname + will be "...svc.". If not specified, the pod will not have a domainname + at all. + type: string + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs to terminate + gracefully. May be decreased in delete request. Value must + be non-negative integer. The value zero indicates delete immediately. + If this value is nil, the default grace period will be used + instead. The grace period is the duration in seconds after + the processes running in the pod are sent a termination signal + and the time when the processes are forcibly halted with a + kill signal. Set this value longer than the expected cleanup + time for your process. Defaults to 30 seconds. + format: int64 + type: integer + tolerations: + description: If specified, the pod's tolerations. + items: + description: The pod this Toleration is attached to tolerates + any taint that matches the triple using + the matching operator . + properties: + effect: + description: Effect indicates the taint effect to match. + Empty means match all taint effects. When specified, + allowed values are NoSchedule, PreferNoSchedule and + NoExecute. + type: string + key: + description: Key is the taint key that the toleration + applies to. Empty means match all taint keys. If the + key is empty, operator must be Exists; this combination + means to match all values and all keys. + type: string + operator: + description: Operator represents a key's relationship + to the value. Valid operators are Exists and Equal. + Defaults to Equal. Exists is equivalent to wildcard + for value, so that a pod can tolerate all taints of + a particular category. + type: string + tolerationSeconds: + description: TolerationSeconds represents the period of + time the toleration (which must be of effect NoExecute, + otherwise this field is ignored) tolerates the taint. + By default, it is not set, which means tolerate the + taint forever (do not evict). Zero and negative values + will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: Value is the taint value the toleration matches + to. If the operator is Exists, the value should be empty, + otherwise just a regular string. + type: string + type: object + type: array + topologySpreadConstraints: + description: TopologySpreadConstraints describes how a group + of pods ought to spread across topology domains. Scheduler + will schedule pods in a way which abides by the constraints. + This field is alpha-level and is only honored by clusters + that enables the EvenPodsSpread feature. All topologySpreadConstraints + are ANDed. + items: + description: TopologySpreadConstraint specifies how to spread + matching pods among the given topology. + properties: + labelSelector: + description: LabelSelector is used to find matching pods. + Pods that match this label selector are counted to determine + the number of pods in their corresponding topology domain. + properties: + matchExpressions: + description: matchExpressions is a list of label selector + requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector + that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector + applies to. + type: string + operator: + description: operator represents a key's relationship + to a set of values. Valid operators are In, + NotIn, Exists and DoesNotExist. + type: string + values: + description: values is an array of string values. + If the operator is In or NotIn, the values + array must be non-empty. If the operator is + Exists or DoesNotExist, the values array must + be empty. This array is replaced during a + strategic merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + A single {key,value} in the matchLabels map is equivalent + to an element of matchExpressions, whose key field + is "key", the operator is "In", and the values array + contains only "value". The requirements are ANDed. + type: object + type: object + maxSkew: + description: 'MaxSkew describes the degree to which pods + may be unevenly distributed. It''s the maximum permitted + difference between the number of matching pods in any + two topology domains of a given topology type. For example, + in a 3-zone cluster, MaxSkew is set to 1, and pods with + the same labelSelector spread as 1/1/0: | zone1 | zone2 + | zone3 | | P | P | | - if MaxSkew is + 1, incoming pod can only be scheduled to zone3 to become + 1/1/1; scheduling it onto zone1(zone2) would make the + ActualSkew(2-0) on zone1(zone2) violate MaxSkew(1). + - if MaxSkew is 2, incoming pod can be scheduled onto + any zone. It''s a required field. Default value is 1 + and 0 is not allowed.' + format: int32 + type: integer + topologyKey: + description: TopologyKey is the key of node labels. Nodes + that have a label with this key and identical values + are considered to be in the same topology. We consider + each as a "bucket", and try to put balanced + number of pods into each bucket. It's a required field. + type: string + whenUnsatisfiable: + description: 'WhenUnsatisfiable indicates how to deal + with a pod if it doesn''t satisfy the spread constraint. + - DoNotSchedule (default) tells the scheduler not to + schedule it - ScheduleAnyway tells the scheduler to + still schedule it It''s considered as "Unsatisfiable" + if and only if placing incoming pod on any topology + violates "MaxSkew". For example, in a 3-zone cluster, + MaxSkew is set to 1, and pods with the same labelSelector + spread as 3/1/1: | zone1 | zone2 | zone3 | | P P P | P | P | + If WhenUnsatisfiable is set to DoNotSchedule, incoming + pod can only be scheduled to zone2(zone3) to become + 3/2/1(3/1/2) as ActualSkew(2-1) on zone2(zone3) satisfies + MaxSkew(1). In other words, the cluster can still be + imbalanced, but scheduler won''t make it *more* imbalanced. + It''s a required field.' + type: string + required: + - maxSkew + - topologyKey + - whenUnsatisfiable + type: object + type: array + x-kubernetes-list-map-keys: + - topologyKey + - whenUnsatisfiable + x-kubernetes-list-type: map + volumes: + description: 'List of volumes that can be mounted by containers + belonging to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes' + items: + description: Volume represents a named volume in a pod that + may be accessed by any container in the pod. + properties: + awsElasticBlockStore: + description: 'AWSElasticBlockStore represents an AWS Disk + resource that is attached to a kubelet''s host machine + and then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + properties: + fsType: + description: 'Filesystem type of the volume that you + want to mount. Tip: Ensure that the filesystem type + is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be + "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore + TODO: how do we prevent errors in the filesystem + from compromising the machine' + type: string + partition: + description: 'The partition in the volume that you + want to mount. If omitted, the default is to mount + by volume name. Examples: For volume /dev/sda1, + you specify the partition as "1". Similarly, the + volume partition for /dev/sda is "0" (or you can + leave the property empty).' + format: int32 + type: integer + readOnly: + description: 'Specify "true" to force and set the + ReadOnly property in VolumeMounts to "true". If + omitted, the default is "false". More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + type: boolean + volumeID: + description: 'Unique ID of the persistent disk resource + in AWS (Amazon EBS volume). More info: https://kubernetes.io/docs/concepts/storage/volumes#awselasticblockstore' + type: string + required: + - volumeID + type: object + azureDisk: + description: AzureDisk represents an Azure Data Disk mount + on the host and bind mount to the pod. + properties: + cachingMode: + description: 'Host Caching mode: None, Read Only, + Read Write.' + type: string + diskName: + description: The Name of the data disk in the blob + storage + type: string + diskURI: + description: The URI the data disk in the blob storage + type: string + fsType: + description: Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. + "ext4", "xfs", "ntfs". Implicitly inferred to be + "ext4" if unspecified. + type: string + kind: + description: 'Expected values Shared: multiple blob + disks per storage account Dedicated: single blob + disk per storage account Managed: azure managed + data disk (only in managed availability set). defaults + to shared' + type: string + readOnly: + description: Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. + type: boolean + required: + - diskName + - diskURI + type: object + azureFile: + description: AzureFile represents an Azure File Service + mount on the host and bind mount to the pod. + properties: + readOnly: + description: Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. + type: boolean + secretName: + description: the name of secret that contains Azure + Storage Account Name and Key + type: string + shareName: + description: Share Name + type: string + required: + - secretName + - shareName + type: object + cephfs: + description: CephFS represents a Ceph FS mount on the + host that shares a pod's lifetime + properties: + monitors: + description: 'Required: Monitors is a collection of + Ceph monitors More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + items: + type: string + type: array + path: + description: 'Optional: Used as the mounted root, + rather than the full Ceph tree, default is /' + type: string + readOnly: + description: 'Optional: Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in + VolumeMounts. More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: boolean + secretFile: + description: 'Optional: SecretFile is the path to + key ring for User, default is /etc/ceph/user.secret + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: string + secretRef: + description: 'Optional: SecretRef is reference to + the authentication secret for User, default is empty. + More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + user: + description: 'Optional: User is the rados user name, + default is admin More info: https://examples.k8s.io/volumes/cephfs/README.md#how-to-use-it' + type: string + required: + - monitors + type: object + cinder: + description: 'Cinder represents a cinder volume attached + and mounted on kubelets host machine. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + properties: + fsType: + description: 'Filesystem type to mount. Must be a + filesystem type supported by the host operating + system. Examples: "ext4", "xfs", "ntfs". Implicitly + inferred to be "ext4" if unspecified. More info: + https://examples.k8s.io/mysql-cinder-pd/README.md' + type: string + readOnly: + description: 'Optional: Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in + VolumeMounts. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: boolean + secretRef: + description: 'Optional: points to a secret object + containing parameters used to connect to OpenStack.' + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + volumeID: + description: 'volume id used to identify the volume + in cinder. More info: https://examples.k8s.io/mysql-cinder-pd/README.md' + type: string + required: + - volumeID + type: object + configMap: + description: ConfigMap represents a configMap that should + populate this volume + properties: + defaultMode: + description: 'Optional: mode bits to use on created + files by default. Must be a value between 0 and + 0777. Defaults to 0644. Directories within the path + are not affected by this setting. This might be + in conflict with other options that affect the file + mode, like fsGroup, and the result can be other + mode bits set.' + format: int32 + type: integer + items: + description: If unspecified, each key-value pair in + the Data field of the referenced ConfigMap will + be projected into the volume as a file whose name + is the key and content is the value. If specified, + the listed keys will be projected into the specified + paths, and unlisted keys will not be present. If + a key is specified which is not present in the ConfigMap, + the volume setup will error unless it is marked + optional. Paths must be relative and may not contain + the '..' path or start with '..'. + items: + description: Maps a string key to a path within + a volume. + properties: + key: + description: The key to project. + type: string + mode: + description: 'Optional: mode bits to use on + this file, must be a value between 0 and 0777. + If not specified, the volume defaultMode will + be used. This might be in conflict with other + options that affect the file mode, like fsGroup, + and the result can be other mode bits set.' + format: int32 + type: integer + path: + description: The relative path of the file to + map the key to. May not be an absolute path. + May not contain the path element '..'. May + not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + optional: + description: Specify whether the ConfigMap or its + keys must be defined + type: boolean + type: object + csi: + description: CSI (Container Storage Interface) represents + storage that is handled by an external CSI driver (Alpha + feature). + properties: + driver: + description: Driver is the name of the CSI driver + that handles this volume. Consult with your admin + for the correct name as registered in the cluster. + type: string + fsType: + description: Filesystem type to mount. Ex. "ext4", + "xfs", "ntfs". If not provided, the empty value + is passed to the associated CSI driver which will + determine the default filesystem to apply. + type: string + nodePublishSecretRef: + description: NodePublishSecretRef is a reference to + the secret object containing sensitive information + to pass to the CSI driver to complete the CSI NodePublishVolume + and NodeUnpublishVolume calls. This field is optional, + and may be empty if no secret is required. If the + secret object contains more than one secret, all + secret references are passed. + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + readOnly: + description: Specifies a read-only configuration for + the volume. Defaults to false (read/write). + type: boolean + volumeAttributes: + additionalProperties: + type: string + description: VolumeAttributes stores driver-specific + properties that are passed to the CSI driver. Consult + your driver's documentation for supported values. + type: object + required: + - driver + type: object + downwardAPI: + description: DownwardAPI represents downward API about + the pod that should populate this volume + properties: + defaultMode: + description: 'Optional: mode bits to use on created + files by default. Must be a value between 0 and + 0777. Defaults to 0644. Directories within the path + are not affected by this setting. This might be + in conflict with other options that affect the file + mode, like fsGroup, and the result can be other + mode bits set.' + format: int32 + type: integer + items: + description: Items is a list of downward API volume + file + items: + description: DownwardAPIVolumeFile represents information + to create the file containing the pod field + properties: + fieldRef: + description: 'Required: Selects a field of the + pod: only annotations, labels, name and namespace + are supported.' + properties: + apiVersion: + description: Version of the schema the FieldPath + is written in terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field to select + in the specified API version. + type: string + required: + - fieldPath + type: object + mode: + description: 'Optional: mode bits to use on + this file, must be a value between 0 and 0777. + If not specified, the volume defaultMode will + be used. This might be in conflict with other + options that affect the file mode, like fsGroup, + and the result can be other mode bits set.' + format: int32 + type: integer + path: + description: 'Required: Path is the relative + path name of the file to be created. Must + not be absolute or contain the ''..'' path. + Must be utf-8 encoded. The first item of the + relative path must not start with ''..''' + type: string + resourceFieldRef: + description: 'Selects a resource of the container: + only resources limits and requests (limits.cpu, + limits.memory, requests.cpu and requests.memory) + are currently supported.' + properties: + containerName: + description: 'Container name: required for + volumes, optional for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output format + of the exposed resources, defaults to + "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource to select' + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + emptyDir: + description: 'EmptyDir represents a temporary directory + that shares a pod''s lifetime. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + properties: + medium: + description: 'What type of storage medium should back + this directory. The default is "" which means to + use the node''s default medium. Must be an empty + string (default) or Memory. More info: https://kubernetes.io/docs/concepts/storage/volumes#emptydir' + type: string + sizeLimit: + anyOf: + - type: integer + - type: string + description: 'Total amount of local storage required + for this EmptyDir volume. The size limit is also + applicable for memory medium. The maximum usage + on memory medium EmptyDir would be the minimum value + between the SizeLimit specified here and the sum + of memory limits of all containers in a pod. The + default is nil which means that the limit is undefined. + More info: http://kubernetes.io/docs/user-guide/volumes#emptydir' + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + fc: + description: FC represents a Fibre Channel resource that + is attached to a kubelet's host machine and then exposed + to the pod. + properties: + fsType: + description: 'Filesystem type to mount. Must be a + filesystem type supported by the host operating + system. Ex. "ext4", "xfs", "ntfs". Implicitly inferred + to be "ext4" if unspecified. TODO: how do we prevent + errors in the filesystem from compromising the machine' + type: string + lun: + description: 'Optional: FC target lun number' + format: int32 + type: integer + readOnly: + description: 'Optional: Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in + VolumeMounts.' + type: boolean + targetWWNs: + description: 'Optional: FC target worldwide names + (WWNs)' + items: + type: string + type: array + wwids: + description: 'Optional: FC volume world wide identifiers + (wwids) Either wwids or combination of targetWWNs + and lun must be set, but not both simultaneously.' + items: + type: string + type: array + type: object + flexVolume: + description: FlexVolume represents a generic volume resource + that is provisioned/attached using an exec based plugin. + properties: + driver: + description: Driver is the name of the driver to use + for this volume. + type: string + fsType: + description: Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. + "ext4", "xfs", "ntfs". The default filesystem depends + on FlexVolume script. + type: string + options: + additionalProperties: + type: string + description: 'Optional: Extra command options if any.' + type: object + readOnly: + description: 'Optional: Defaults to false (read/write). + ReadOnly here will force the ReadOnly setting in + VolumeMounts.' + type: boolean + secretRef: + description: 'Optional: SecretRef is reference to + the secret object containing sensitive information + to pass to the plugin scripts. This may be empty + if no secret object is specified. If the secret + object contains more than one secret, all secrets + are passed to the plugin scripts.' + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + required: + - driver + type: object + flocker: + description: Flocker represents a Flocker volume attached + to a kubelet's host machine. This depends on the Flocker + control service being running + properties: + datasetName: + description: Name of the dataset stored as metadata + -> name on the dataset for Flocker should be considered + as deprecated + type: string + datasetUUID: + description: UUID of the dataset. This is unique identifier + of a Flocker dataset + type: string + type: object + gcePersistentDisk: + description: 'GCEPersistentDisk represents a GCE Disk + resource that is attached to a kubelet''s host machine + and then exposed to the pod. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + properties: + fsType: + description: 'Filesystem type of the volume that you + want to mount. Tip: Ensure that the filesystem type + is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be + "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk + TODO: how do we prevent errors in the filesystem + from compromising the machine' + type: string + partition: + description: 'The partition in the volume that you + want to mount. If omitted, the default is to mount + by volume name. Examples: For volume /dev/sda1, + you specify the partition as "1". Similarly, the + volume partition for /dev/sda is "0" (or you can + leave the property empty). More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + format: int32 + type: integer + pdName: + description: 'Unique name of the PD resource in GCE. + Used to identify the disk in GCE. More info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + type: string + readOnly: + description: 'ReadOnly here will force the ReadOnly + setting in VolumeMounts. Defaults to false. More + info: https://kubernetes.io/docs/concepts/storage/volumes#gcepersistentdisk' + type: boolean + required: + - pdName + type: object + gitRepo: + description: 'GitRepo represents a git repository at a + particular revision. DEPRECATED: GitRepo is deprecated. + To provision a container with a git repo, mount an EmptyDir + into an InitContainer that clones the repo using git, + then mount the EmptyDir into the Pod''s container.' + properties: + directory: + description: Target directory name. Must not contain + or start with '..'. If '.' is supplied, the volume + directory will be the git repository. Otherwise, + if specified, the volume will contain the git repository + in the subdirectory with the given name. + type: string + repository: + description: Repository URL + type: string + revision: + description: Commit hash for the specified revision. + type: string + required: + - repository + type: object + glusterfs: + description: 'Glusterfs represents a Glusterfs mount on + the host that shares a pod''s lifetime. More info: https://examples.k8s.io/volumes/glusterfs/README.md' + properties: + endpoints: + description: 'EndpointsName is the endpoint name that + details Glusterfs topology. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: string + path: + description: 'Path is the Glusterfs volume path. More + info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: string + readOnly: + description: 'ReadOnly here will force the Glusterfs + volume to be mounted with read-only permissions. + Defaults to false. More info: https://examples.k8s.io/volumes/glusterfs/README.md#create-a-pod' + type: boolean + required: + - endpoints + - path + type: object + hostPath: + description: 'HostPath represents a pre-existing file + or directory on the host machine that is directly exposed + to the container. This is generally used for system + agents or other privileged things that are allowed to + see the host machine. Most containers will NOT need + this. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath + --- TODO(jonesdl) We need to restrict who can use host + directory mounts and who can/can not mount host directories + as read/write.' + properties: + path: + description: 'Path of the directory on the host. If + the path is a symlink, it will follow the link to + the real path. More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' + type: string + type: + description: 'Type for HostPath Volume Defaults to + "" More info: https://kubernetes.io/docs/concepts/storage/volumes#hostpath' + type: string + required: + - path + type: object + iscsi: + description: 'ISCSI represents an ISCSI Disk resource + that is attached to a kubelet''s host machine and then + exposed to the pod. More info: https://examples.k8s.io/volumes/iscsi/README.md' + properties: + chapAuthDiscovery: + description: whether support iSCSI Discovery CHAP + authentication + type: boolean + chapAuthSession: + description: whether support iSCSI Session CHAP authentication + type: boolean + fsType: + description: 'Filesystem type of the volume that you + want to mount. Tip: Ensure that the filesystem type + is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be + "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#iscsi + TODO: how do we prevent errors in the filesystem + from compromising the machine' + type: string + initiatorName: + description: Custom iSCSI Initiator Name. If initiatorName + is specified with iscsiInterface simultaneously, + new iSCSI interface : + will be created for the connection. + type: string + iqn: + description: Target iSCSI Qualified Name. + type: string + iscsiInterface: + description: iSCSI Interface Name that uses an iSCSI + transport. Defaults to 'default' (tcp). + type: string + lun: + description: iSCSI Target Lun number. + format: int32 + type: integer + portals: + description: iSCSI Target Portal List. The portal + is either an IP or ip_addr:port if the port is other + than default (typically TCP ports 860 and 3260). + items: + type: string + type: array + readOnly: + description: ReadOnly here will force the ReadOnly + setting in VolumeMounts. Defaults to false. + type: boolean + secretRef: + description: CHAP Secret for iSCSI target and initiator + authentication + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + targetPortal: + description: iSCSI Target Portal. The Portal is either + an IP or ip_addr:port if the port is other than + default (typically TCP ports 860 and 3260). + type: string + required: + - iqn + - lun + - targetPortal + type: object + name: + description: 'Volume''s name. Must be a DNS_LABEL and + unique within the pod. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names' + type: string + nfs: + description: 'NFS represents an NFS mount on the host + that shares a pod''s lifetime More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + properties: + path: + description: 'Path that is exported by the NFS server. + More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: string + readOnly: + description: 'ReadOnly here will force the NFS export + to be mounted with read-only permissions. Defaults + to false. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: boolean + server: + description: 'Server is the hostname or IP address + of the NFS server. More info: https://kubernetes.io/docs/concepts/storage/volumes#nfs' + type: string + required: + - path + - server + type: object + persistentVolumeClaim: + description: 'PersistentVolumeClaimVolumeSource represents + a reference to a PersistentVolumeClaim in the same namespace. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + properties: + claimName: + description: 'ClaimName is the name of a PersistentVolumeClaim + in the same namespace as the pod using this volume. + More info: https://kubernetes.io/docs/concepts/storage/persistent-volumes#persistentvolumeclaims' + type: string + readOnly: + description: Will force the ReadOnly setting in VolumeMounts. + Default false. + type: boolean + required: + - claimName + type: object + photonPersistentDisk: + description: PhotonPersistentDisk represents a PhotonController + persistent disk attached and mounted on kubelets host + machine + properties: + fsType: + description: Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. + "ext4", "xfs", "ntfs". Implicitly inferred to be + "ext4" if unspecified. + type: string + pdID: + description: ID that identifies Photon Controller + persistent disk + type: string + required: + - pdID + type: object + portworxVolume: + description: PortworxVolume represents a portworx volume + attached and mounted on kubelets host machine + properties: + fsType: + description: FSType represents the filesystem type + to mount Must be a filesystem type supported by + the host operating system. Ex. "ext4", "xfs". Implicitly + inferred to be "ext4" if unspecified. + type: string + readOnly: + description: Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. + type: boolean + volumeID: + description: VolumeID uniquely identifies a Portworx + volume + type: string + required: + - volumeID + type: object + projected: + description: Items for all in one resources secrets, configmaps, + and downward API + properties: + defaultMode: + description: Mode bits to use on created files by + default. Must be a value between 0 and 0777. Directories + within the path are not affected by this setting. + This might be in conflict with other options that + affect the file mode, like fsGroup, and the result + can be other mode bits set. + format: int32 + type: integer + sources: + description: list of volume projections + items: + description: Projection that may be projected along + with other supported volume types + properties: + configMap: + description: information about the configMap + data to project + properties: + items: + description: If unspecified, each key-value + pair in the Data field of the referenced + ConfigMap will be projected into the volume + as a file whose name is the key and content + is the value. If specified, the listed + keys will be projected into the specified + paths, and unlisted keys will not be present. + If a key is specified which is not present + in the ConfigMap, the volume setup will + error unless it is marked optional. Paths + must be relative and may not contain the + '..' path or start with '..'. + items: + description: Maps a string key to a path + within a volume. + properties: + key: + description: The key to project. + type: string + mode: + description: 'Optional: mode bits + to use on this file, must be a value + between 0 and 0777. If not specified, + the volume defaultMode will be used. + This might be in conflict with other + options that affect the file mode, + like fsGroup, and the result can + be other mode bits set.' + format: int32 + type: integer + path: + description: The relative path of + the file to map the key to. May + not be an absolute path. May not + contain the path element '..'. May + not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the ConfigMap + or its keys must be defined + type: boolean + type: object + downwardAPI: + description: information about the downwardAPI + data to project + properties: + items: + description: Items is a list of DownwardAPIVolume + file + items: + description: DownwardAPIVolumeFile represents + information to create the file containing + the pod field + properties: + fieldRef: + description: 'Required: Selects a + field of the pod: only annotations, + labels, name and namespace are supported.' + properties: + apiVersion: + description: Version of the schema + the FieldPath is written in + terms of, defaults to "v1". + type: string + fieldPath: + description: Path of the field + to select in the specified API + version. + type: string + required: + - fieldPath + type: object + mode: + description: 'Optional: mode bits + to use on this file, must be a value + between 0 and 0777. If not specified, + the volume defaultMode will be used. + This might be in conflict with other + options that affect the file mode, + like fsGroup, and the result can + be other mode bits set.' + format: int32 + type: integer + path: + description: 'Required: Path is the + relative path name of the file to + be created. Must not be absolute + or contain the ''..'' path. Must + be utf-8 encoded. The first item + of the relative path must not start + with ''..''' + type: string + resourceFieldRef: + description: 'Selects a resource of + the container: only resources limits + and requests (limits.cpu, limits.memory, + requests.cpu and requests.memory) + are currently supported.' + properties: + containerName: + description: 'Container name: + required for volumes, optional + for env vars' + type: string + divisor: + anyOf: + - type: integer + - type: string + description: Specifies the output + format of the exposed resources, + defaults to "1" + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + description: 'Required: resource + to select' + type: string + required: + - resource + type: object + required: + - path + type: object + type: array + type: object + secret: + description: information about the secret data + to project + properties: + items: + description: If unspecified, each key-value + pair in the Data field of the referenced + Secret will be projected into the volume + as a file whose name is the key and content + is the value. If specified, the listed + keys will be projected into the specified + paths, and unlisted keys will not be present. + If a key is specified which is not present + in the Secret, the volume setup will error + unless it is marked optional. Paths must + be relative and may not contain the '..' + path or start with '..'. + items: + description: Maps a string key to a path + within a volume. + properties: + key: + description: The key to project. + type: string + mode: + description: 'Optional: mode bits + to use on this file, must be a value + between 0 and 0777. If not specified, + the volume defaultMode will be used. + This might be in conflict with other + options that affect the file mode, + like fsGroup, and the result can + be other mode bits set.' + format: int32 + type: integer + path: + description: The relative path of + the file to map the key to. May + not be an absolute path. May not + contain the path element '..'. May + not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + name: + description: 'Name of the referent. More + info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, + kind, uid?' + type: string + optional: + description: Specify whether the Secret + or its key must be defined + type: boolean + type: object + serviceAccountToken: + description: information about the serviceAccountToken + data to project + properties: + audience: + description: Audience is the intended audience + of the token. A recipient of a token must + identify itself with an identifier specified + in the audience of the token, and otherwise + should reject the token. The audience + defaults to the identifier of the apiserver. + type: string + expirationSeconds: + description: ExpirationSeconds is the requested + duration of validity of the service account + token. As the token approaches expiration, + the kubelet volume plugin will proactively + rotate the service account token. The + kubelet will start trying to rotate the + token if the token is older than 80 percent + of its time to live or if the token is + older than 24 hours.Defaults to 1 hour + and must be at least 10 minutes. + format: int64 + type: integer + path: + description: Path is the path relative to + the mount point of the file to project + the token into. + type: string + required: + - path + type: object + type: object + type: array + required: + - sources + type: object + quobyte: + description: Quobyte represents a Quobyte mount on the + host that shares a pod's lifetime + properties: + group: + description: Group to map volume access to Default + is no group + type: string + readOnly: + description: ReadOnly here will force the Quobyte + volume to be mounted with read-only permissions. + Defaults to false. + type: boolean + registry: + description: Registry represents a single or multiple + Quobyte Registry services specified as a string + as host:port pair (multiple entries are separated + with commas) which acts as the central registry + for volumes + type: string + tenant: + description: Tenant owning the given Quobyte volume + in the Backend Used with dynamically provisioned + Quobyte volumes, value is set by the plugin + type: string + user: + description: User to map volume access to Defaults + to serivceaccount user + type: string + volume: + description: Volume is a string that references an + already created Quobyte volume by name. + type: string + required: + - registry + - volume + type: object + rbd: + description: 'RBD represents a Rados Block Device mount + on the host that shares a pod''s lifetime. More info: + https://examples.k8s.io/volumes/rbd/README.md' + properties: + fsType: + description: 'Filesystem type of the volume that you + want to mount. Tip: Ensure that the filesystem type + is supported by the host operating system. Examples: + "ext4", "xfs", "ntfs". Implicitly inferred to be + "ext4" if unspecified. More info: https://kubernetes.io/docs/concepts/storage/volumes#rbd + TODO: how do we prevent errors in the filesystem + from compromising the machine' + type: string + image: + description: 'The rados image name. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + keyring: + description: 'Keyring is the path to key ring for + RBDUser. Default is /etc/ceph/keyring. More info: + https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + monitors: + description: 'A collection of Ceph monitors. More + info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + items: + type: string + type: array + pool: + description: 'The rados pool name. Default is rbd. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + readOnly: + description: 'ReadOnly here will force the ReadOnly + setting in VolumeMounts. Defaults to false. More + info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: boolean + secretRef: + description: 'SecretRef is name of the authentication + secret for RBDUser. If provided overrides keyring. + Default is nil. More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + user: + description: 'The rados user name. Default is admin. + More info: https://examples.k8s.io/volumes/rbd/README.md#how-to-use-it' + type: string + required: + - image + - monitors + type: object + scaleIO: + description: ScaleIO represents a ScaleIO persistent volume + attached and mounted on Kubernetes nodes. + properties: + fsType: + description: Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. + "ext4", "xfs", "ntfs". Default is "xfs". + type: string + gateway: + description: The host address of the ScaleIO API Gateway. + type: string + protectionDomain: + description: The name of the ScaleIO Protection Domain + for the configured storage. + type: string + readOnly: + description: Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: SecretRef references to the secret for + ScaleIO user and other sensitive information. If + this is not provided, Login operation will fail. + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + sslEnabled: + description: Flag to enable/disable SSL communication + with Gateway, default false + type: boolean + storageMode: + description: Indicates whether the storage for a volume + should be ThickProvisioned or ThinProvisioned. Default + is ThinProvisioned. + type: string + storagePool: + description: The ScaleIO Storage Pool associated with + the protection domain. + type: string + system: + description: The name of the storage system as configured + in ScaleIO. + type: string + volumeName: + description: The name of a volume already created + in the ScaleIO system that is associated with this + volume source. + type: string + required: + - gateway + - secretRef + - system + type: object + secret: + description: 'Secret represents a secret that should populate + this volume. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' + properties: + defaultMode: + description: 'Optional: mode bits to use on created + files by default. Must be a value between 0 and + 0777. Defaults to 0644. Directories within the path + are not affected by this setting. This might be + in conflict with other options that affect the file + mode, like fsGroup, and the result can be other + mode bits set.' + format: int32 + type: integer + items: + description: If unspecified, each key-value pair in + the Data field of the referenced Secret will be + projected into the volume as a file whose name is + the key and content is the value. If specified, + the listed keys will be projected into the specified + paths, and unlisted keys will not be present. If + a key is specified which is not present in the Secret, + the volume setup will error unless it is marked + optional. Paths must be relative and may not contain + the '..' path or start with '..'. + items: + description: Maps a string key to a path within + a volume. + properties: + key: + description: The key to project. + type: string + mode: + description: 'Optional: mode bits to use on + this file, must be a value between 0 and 0777. + If not specified, the volume defaultMode will + be used. This might be in conflict with other + options that affect the file mode, like fsGroup, + and the result can be other mode bits set.' + format: int32 + type: integer + path: + description: The relative path of the file to + map the key to. May not be an absolute path. + May not contain the path element '..'. May + not start with the string '..'. + type: string + required: + - key + - path + type: object + type: array + optional: + description: Specify whether the Secret or its keys + must be defined + type: boolean + secretName: + description: 'Name of the secret in the pod''s namespace + to use. More info: https://kubernetes.io/docs/concepts/storage/volumes#secret' + type: string + type: object + storageos: + description: StorageOS represents a StorageOS volume attached + and mounted on Kubernetes nodes. + properties: + fsType: + description: Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. + "ext4", "xfs", "ntfs". Implicitly inferred to be + "ext4" if unspecified. + type: string + readOnly: + description: Defaults to false (read/write). ReadOnly + here will force the ReadOnly setting in VolumeMounts. + type: boolean + secretRef: + description: SecretRef specifies the secret to use + for obtaining the StorageOS API credentials. If + not specified, default values will be attempted. + properties: + name: + description: 'Name of the referent. More info: + https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names + TODO: Add other useful fields. apiVersion, kind, + uid?' + type: string + type: object + volumeName: + description: VolumeName is the human-readable name + of the StorageOS volume. Volume names are only + unique within a namespace. + type: string + volumeNamespace: + description: VolumeNamespace specifies the scope of + the volume within StorageOS. If no namespace is + specified then the Pod's namespace will be used. This + allows the Kubernetes name scoping to be mirrored + within StorageOS for tighter integration. Set VolumeName + to any name to override the default behaviour. Set + to "default" if you are not using namespaces within + StorageOS. Namespaces that do not pre-exist within + StorageOS will be created. + type: string + type: object + vsphereVolume: + description: VsphereVolume represents a vSphere volume + attached and mounted on kubelets host machine + properties: + fsType: + description: Filesystem type to mount. Must be a filesystem + type supported by the host operating system. Ex. + "ext4", "xfs", "ntfs". Implicitly inferred to be + "ext4" if unspecified. + type: string + storagePolicyID: + description: Storage Policy Based Management (SPBM) + profile ID associated with the StoragePolicyName. + type: string + storagePolicyName: + description: Storage Policy Based Management (SPBM) + profile name. + type: string + volumePath: + description: Path that identifies vSphere volume vmdk + type: string + required: + - volumePath + type: object + required: + - name + type: object + type: array + required: + - containers + type: object + type: object + required: + - selector + - template + type: object + status: + description: DaemonJobStatus defines the observed state of DaemonJob + properties: + collisionCount: + description: Count of hash collisions for the DaemonSet. The DaemonSet + controller uses this field as a collision avoidance mechanism + when it needs to create the name for the newest ControllerRevision. + format: int32 + type: integer + conditions: + description: Represents the latest available observations of a DaemonSet's + current state. + items: + type: object + properties: + lastTransitionTime: + description: Last time the condition transitioned from one + status to another. + format: date-time + type: string + message: + description: A human readable message indicating details about + the transition. + type: string + reason: + description: The reason for the condition's last transition. + type: string + status: + description: Status of the condition, one of True, False, + Unknown. + type: string + type: + description: Type of DaemonSet condition. + type: string + type: array + currentNumberScheduled: + description: 'The number of nodes that are running at least 1 daemon + pod and are supposed to run the daemon pod. More info: https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/' + format: int32 + type: integer + desiredNumberScheduled: + description: 'The total number of nodes that should be running the + daemon pod (including nodes correctly running the daemon pod). + More info: https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/' + format: int32 + type: integer + numberAvailable: + description: The number of nodes that should be running the daemon + pod and have one or more of the daemon pod running and available + (ready for at least spec.minReadySeconds) + format: int32 + type: integer + numberMisscheduled: + description: 'The number of nodes that are running the daemon pod, + but are not supposed to run the daemon pod. More info: https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/' + format: int32 + type: integer + numberReady: + description: The number of nodes that should be running the daemon + pod and have one or more of the daemon pod running and ready. + format: int32 + type: integer + numberUnavailable: + description: The number of nodes that should be running the daemon + pod and have none of the daemon pod running and available (ready + for at least spec.minReadySeconds) + format: int32 + type: integer + observedGeneration: + description: The most recent generation observed by the daemon set + controller. + format: int64 + type: integer + updatedNumberScheduled: + description: The total number of nodes that are running updated + daemon pod + format: int32 + type: integer + type: object + type: object + subresources: + status: {} + scope: Namespaced + names: + plural: daemonjobs + singular: daemonjob + kind: DaemonJob + shortNames: ["dj"] +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +{{- end }} diff --git a/daemonjob-controller/templates/deployment.yaml b/daemonjob-controller/templates/deployment.yaml new file mode 100644 index 0000000000..177ed0f4c7 --- /dev/null +++ b/daemonjob-controller/templates/deployment.yaml @@ -0,0 +1,63 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.deployment }} +{{- $envAll := . }} + +{{- $serviceAccountName := "daemonjob-controller-serviceaccount" }} +{{ tuple $envAll "daemonjob_controller" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: daemonjob-controller + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 4 }} + namespace: {{ .Release.Namespace }} + labels: +{{ tuple $envAll "daemonjob-controller" "controller" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + replicas: {{ .Values.pod.replicas.daemonjob_controller }} + selector: + matchLabels: +{{ tuple $envAll "daemonjob-controller" "controller" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} + template: + metadata: + annotations: +{{ dict "envAll" $envAll "podName" "daemonjob-controller" "containerNames" (list "controller") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + labels: +{{ tuple $envAll "daemonjob-controller" "controller" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + spec: + serviceAccountName: {{ $serviceAccountName }} + nodeSelector: + {{ .Values.labels.daemonjob_controller.node_selector_key }}: {{ .Values.labels.daemonjob_controller.node_selector_value | quote }} + containers: + - name: controller +{{ tuple $envAll "python" | include "helm-toolkit.snippets.image" | indent 8 }} +{{ tuple $envAll $envAll.Values.pod.resources.daemonjob_controller | include "helm-toolkit.snippets.kubernetes_resources" | indent 8 }} +{{ dict "envAll" $envAll "application" "daemonjob_controller" "container" "controller" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 8 }} + command: + - python + - /hooks/sync.py + volumeMounts: + - name: hooks + mountPath: /hooks + readOnly: true + volumes: + - name: hooks + configMap: + name: daemonjob-controller-bin + defaultMode: 0555 +{{- end }} diff --git a/daemonjob-controller/templates/job-image-repo-sync.yaml b/daemonjob-controller/templates/job-image-repo-sync.yaml new file mode 100644 index 0000000000..b8a37270c6 --- /dev/null +++ b/daemonjob-controller/templates/job-image-repo-sync.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "daemonjob-controller" -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} \ No newline at end of file diff --git a/daemonjob-controller/templates/secret-registry.yaml b/daemonjob-controller/templates/secret-registry.yaml new file mode 100644 index 0000000000..da979b3223 --- /dev/null +++ b/daemonjob-controller/templates/secret-registry.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_registry .Values.endpoints.oci_image_registry.auth.enabled }} +{{ include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) }} +{{- end }} diff --git a/daemonjob-controller/templates/service.yaml b/daemonjob-controller/templates/service.yaml new file mode 100644 index 0000000000..2e87db9596 --- /dev/null +++ b/daemonjob-controller/templates/service.yaml @@ -0,0 +1,28 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.service }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "daemonjob_controller" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + namespace: {{ .Release.Namespace }} +spec: + ports: + - port: 80 + selector: +{{ tuple $envAll "daemonjob-controller" "controller" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- end }} \ No newline at end of file diff --git a/daemonjob-controller/values.yaml b/daemonjob-controller/values.yaml new file mode 100644 index 0000000000..5ae11e76c5 --- /dev/null +++ b/daemonjob-controller/values.yaml @@ -0,0 +1,135 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for elasticsearch +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +--- +release_group: null + +images: + tags: + python: docker.io/library/python:3.7-slim + pause: registry.k8s.io/pause:latest + image_repo_sync: docker.io/library/docker:17.07.0 + pullPolicy: IfNotPresent + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +labels: + daemonjob_controller: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +crds: + group_name: ctl.example.com + group_version: v1 + +pod: + lifecycle: + upgrades: + deployments: + pod_replacement_strategy: RollingUpdate + revision_history: 3 + rolling_update: + max_surge: 3 + max_unavailable: 1 + resources: + enabled: false + daemonjob_controller: + limits: + memory: "1024Mi" + cpu: "2000m" + requests: + memory: "128Mi" + cpu: "500m" + replicas: + daemonjob_controller: 1 + security_context: + daemonjob_controller: + pod: + runAsUser: 34356 + runAsNonRoot: true + container: + controller: + runAsUser: 0 + readOnlyRootFilesystem: true +secrets: + oci_image_registry: + daemonjob-controller: daemonjob-controller-oci-image-registry-key +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + daemonjob-controller: + username: daemonjob-controller + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + daemonjob_controller: + hosts: + default: daemonjob-controller + host_fqdn_override: + default: null + port: + http: + default: 80 + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - daemonjob-controller-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + daemonjob_controller: + services: null + +manifests: + deployment: true + crds_create: true + job_image_repo_sync: true + configmap_bin: true + secret_registry: true + service: true +... diff --git a/elastic-apm-server/Chart.yaml b/elastic-apm-server/Chart.yaml new file mode 100644 index 0000000000..c5c7a66c64 --- /dev/null +++ b/elastic-apm-server/Chart.yaml @@ -0,0 +1,29 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: v6.2.3 +description: OpenStack-Helm Elastic APM Server +name: elastic-apm-server +version: 2024.2.0 +home: https://www.elastic.co/guide/en/apm/get-started/current/index.html +sources: + - https://github.com/elastic/apm-server + - https://opendev.org/openstack/openstack-helm-infra +maintainers: + - name: OpenStack-Helm Authors +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit/ + version: ">= 0.1.0" +... diff --git a/elastic-apm-server/templates/configmap-bin.yaml b/elastic-apm-server/templates/configmap-bin.yaml new file mode 100644 index 0000000000..0535dd106c --- /dev/null +++ b/elastic-apm-server/templates/configmap-bin.yaml @@ -0,0 +1,25 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_bin }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: elastic-apm-server-bin +data: + image-repo-sync.sh: | +{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} +{{- end }} diff --git a/elastic-apm-server/templates/configmap-etc.yaml b/elastic-apm-server/templates/configmap-etc.yaml new file mode 100644 index 0000000000..e405b22e71 --- /dev/null +++ b/elastic-apm-server/templates/configmap-etc.yaml @@ -0,0 +1,25 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_etc }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: elastic-apm-server-etc +data: + apm-server.yml: | +{{ toYaml .Values.conf.apm_server | indent 4 }} +{{- end }} diff --git a/elastic-apm-server/templates/deployment.yaml b/elastic-apm-server/templates/deployment.yaml new file mode 100644 index 0000000000..be1f5bf83c --- /dev/null +++ b/elastic-apm-server/templates/deployment.yaml @@ -0,0 +1,132 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.deployment }} +{{- $envAll := . }} +{{- $esUserSecret := .Values.secrets.elasticsearch.user }} + +{{- $mounts_elastic_apm_server := .Values.pod.mounts.elastic_apm_server.elastic_apm_server }} + +{{- $serviceAccountName := printf "%s-%s" .Release.Name "elastic-apm-server" }} +{{ tuple $envAll "elastic-apm-server" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ $serviceAccountName }} + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ $serviceAccountName }} +rules: +- apiGroups: [""] + resources: + - namespaces + - pods + verbs: + - get + - list + - watch +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: elastic-apm-server + labels: +{{ tuple $envAll "elastic-apm-server" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + selector: + matchLabels: +{{ tuple $envAll "elastic-apm-server" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} +{{ tuple $envAll "elastic-apm-server" | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "elastic-apm-server" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} +{{ dict "envAll" $envAll "podName" "elastic-apm-server" "containerNames" (list "elastic-apm-server" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: + replicas: {{ .Values.pod.replicas.elastic_apm_server }} + serviceAccountName: {{ $serviceAccountName }} + nodeSelector: + {{ .Values.labels.elastic_apm_server.node_selector_key }}: {{ .Values.labels.elastic_apm_server.node_selector_value }} + initContainers: +{{ tuple $envAll "elastic_apm_server" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: elastic-apm-server + image: {{ .Values.images.tags.elastic_apm_server }} + imagePullPolicy: {{ .Values.images.pull_policy }} + securityContext: + runAsUser: 0 +{{ tuple $envAll $envAll.Values.pod.resources.elastic_apm_server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + args: + - "-c" + - "/usr/share/apm-server/apm-server.yml" + - "-e" + ports: + - name: server + containerPort: {{ tuple "elastic_apm_server" "internal" "server" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + env: + - name: ELASTICSEARCH_HOST + value: {{ tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }} + - name: ELASTICSEARCH_PORT + value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: APM_SERVER_HOST + value: {{ tuple "elastic_apm_server" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }} + - name: APM_SERVER_PORT + value: {{ tuple "elastic_apm_server" "internal" "server" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: ELASTICSEARCH_USERNAME + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_USERNAME + - name: ELASTICSEARCH_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_PASSWORD + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: elastic-apm-server-etc + mountPath: /usr/share/apm-server/apm-server.yml + readOnly: true + subPath: apm-server.yml + - name: data + mountPath: /usr/share/apm-server/data +{{ if $mounts_elastic_apm_server.volumeMounts }}{{ toYaml $mounts_elastic_apm_server.volumeMounts | indent 8 }}{{ end }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: elastic-apm-server-etc + configMap: + name: elastic-apm-server-etc + defaultMode: 0444 + - name: data + hostPath: + path: /var/lib/elastic-apm-server + type: DirectoryOrCreate +{{ if $mounts_elastic_apm_server.volumes }}{{ toYaml $mounts_elastic_apm_server.volumes | indent 8 }}{{ end }} +{{- end }} diff --git a/elastic-apm-server/templates/job-image-repo-sync.yaml b/elastic-apm-server/templates/job-image-repo-sync.yaml new file mode 100644 index 0000000000..8502f2b60b --- /dev/null +++ b/elastic-apm-server/templates/job-image-repo-sync.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "filebeat" -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/elastic-apm-server/templates/secret-elasticsearch-creds.yaml b/elastic-apm-server/templates/secret-elasticsearch-creds.yaml new file mode 100644 index 0000000000..347eaa9d0f --- /dev/null +++ b/elastic-apm-server/templates/secret-elasticsearch-creds.yaml @@ -0,0 +1,27 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_elasticsearch }} +{{- $envAll := . }} +{{- $secretName := index $envAll.Values.secrets.elasticsearch.user }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} +type: Opaque +data: + ELASTICSEARCH_USERNAME: {{ .Values.endpoints.elasticsearch.auth.admin.username | b64enc }} + ELASTICSEARCH_PASSWORD: {{ .Values.endpoints.elasticsearch.auth.admin.password | b64enc }} +{{- end }} diff --git a/elastic-apm-server/templates/secret-registry.yaml b/elastic-apm-server/templates/secret-registry.yaml new file mode 100644 index 0000000000..da979b3223 --- /dev/null +++ b/elastic-apm-server/templates/secret-registry.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_registry .Values.endpoints.oci_image_registry.auth.enabled }} +{{ include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) }} +{{- end }} diff --git a/elastic-apm-server/templates/service.yaml b/elastic-apm-server/templates/service.yaml new file mode 100644 index 0000000000..2e917444c0 --- /dev/null +++ b/elastic-apm-server/templates/service.yaml @@ -0,0 +1,32 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "elastic_apm_server" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +spec: + ports: + - name: server + port: {{ tuple "elastic_apm_server" "internal" "server" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + {{ if .Values.network.elastic_apm_server.node_port.enabled }} + nodePort: {{ .Values.network.elastic_apm_server.node_port.port }} + {{ end }} + selector: +{{ tuple $envAll "elastic-apm-server" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + {{ if .Values.network.elastic_apm_server.node_port.enabled }} + type: NodePort + {{ end }} diff --git a/elastic-apm-server/values.yaml b/elastic-apm-server/values.yaml new file mode 100644 index 0000000000..e728bc7d9c --- /dev/null +++ b/elastic-apm-server/values.yaml @@ -0,0 +1,184 @@ +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for elastic-apm-server +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +--- +release_group: null + +labels: + elastic_apm_server: + node_selector_key: openstack-control-plane + node_selector_value: enabled + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +images: + tags: + elastic_apm_server: docker.elastic.co/apm/apm-server:6.2.3 + dep_check: quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal + image_repo_sync: docker.io/library/docker:17.07.0 + pull_policy: IfNotPresent + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +secrets: + elasticsearch: + user: elastic-apm-server-elasticsearch-user + oci_image_registry: + elastic-apm-server: elastic-apm-server-oci-image-registry + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - elastic-apm-server-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + elastic_apm_server: + services: null + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + +conf: + apm_server: + setup: + dashboards: + enabled: true + host: ['${APM_SERVER_HOST}:${APM_SERVER_PORT}'] + output: + elasticsearch: + hosts: ["${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}"] + username: "${ELASTICSEARCH_USERNAME}" + password: "${ELASTICSEARCH_PASSWORD}" + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + elastic-apm-server: + username: elastic-apm-server + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + elasticsearch: + namespace: null + name: elasticsearch + auth: + admin: + username: admin + password: changeme + hosts: + data: elasticsearch-data + default: elasticsearch-logging + discovery: elasticsearch-discovery + public: elasticsearch + host_fqdn_override: + default: null + path: + default: null + scheme: + default: http + port: + http: + default: 80 + elastic_apm_server: + namespace: null + name: apm-server + hosts: + default: apm-server + host_fqdn_override: + default: null + path: + default: null + scheme: + default: http + port: + server: + default: 8200 + +pod: + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + lifecycle: + upgrades: + daemonsets: + pod_replacement_strategy: RollingUpdate + elastic_apm_server: + enabled: true + min_ready_seconds: 0 + max_unavailable: 1 + replicas: + elastic_apm_server: 1 + resources: + elastic_apm_server: + enabled: false + limits: + memory: '400Mi' + cpu: '400m' + requests: + memory: '100Mi' + cpu: '100m' + mounts: + elastic_apm_server: + elastic_apm_server: + +network: + elastic_apm_server: + node_port: + enabled: false + port: 30200 + +manifests: + configmap_bin: true + configmap_etc: true + deployment: true + service: true + job_image_repo_sync: true + secret_elasticsearch: true + secret_registry: true +... diff --git a/elastic-filebeat/Chart.yaml b/elastic-filebeat/Chart.yaml new file mode 100644 index 0000000000..864d584100 --- /dev/null +++ b/elastic-filebeat/Chart.yaml @@ -0,0 +1,29 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: v7.1.0 +description: OpenStack-Helm Elastic Filebeat +name: elastic-filebeat +version: 2024.2.0 +home: https://www.elastic.co/products/beats/filebeat +sources: + - https://github.com/elastic/beats/tree/master/filebeat + - https://opendev.org/openstack/openstack-helm-infra +maintainers: + - name: OpenStack-Helm Authors +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit/ + version: ">= 0.1.0" +... diff --git a/elastic-filebeat/templates/configmap-bin.yaml b/elastic-filebeat/templates/configmap-bin.yaml new file mode 100644 index 0000000000..432c49a78c --- /dev/null +++ b/elastic-filebeat/templates/configmap-bin.yaml @@ -0,0 +1,25 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_bin }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: filebeat-bin +data: + image-repo-sync.sh: | +{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} +{{- end }} diff --git a/elastic-filebeat/templates/configmap-etc.yaml b/elastic-filebeat/templates/configmap-etc.yaml new file mode 100644 index 0000000000..bc32bf4019 --- /dev/null +++ b/elastic-filebeat/templates/configmap-etc.yaml @@ -0,0 +1,27 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_etc }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: filebeat-etc +data: + filebeat.yml: | +{{ toYaml .Values.conf.filebeat | indent 4 }} + system.yml: | +{{ toYaml .Values.conf.modules.system | indent 4 }} +{{- end }} diff --git a/elastic-filebeat/templates/daemonset.yaml b/elastic-filebeat/templates/daemonset.yaml new file mode 100644 index 0000000000..cc0c7c75b6 --- /dev/null +++ b/elastic-filebeat/templates/daemonset.yaml @@ -0,0 +1,167 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.daemonset }} +{{- $envAll := . }} +{{- $esUserSecret := .Values.secrets.elasticsearch.user }} + +{{- $mounts_filebeat := .Values.pod.mounts.filebeat.filebeat }} + +{{- $serviceAccountName := printf "%s-%s" .Release.Name "filebeat" }} +{{ tuple $envAll "filebeat" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ $serviceAccountName }} + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ $serviceAccountName }} +rules: + - apiGroups: + - "" + resources: + - namespaces + - nodes + - pods + - services + - endpoints + - replicationcontrollers + - limitranges + verbs: + - get + - list + - watch + - apiGroups: + - apps + resources: + - statefulsets + - daemonsets + - deployments + - replicasets + verbs: + - get + - list + - watch +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: filebeat +spec: + selector: + matchLabels: +{{ tuple $envAll "filebeat" "daemon" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} +{{ tuple $envAll "filebeat" | include "helm-toolkit.snippets.kubernetes_upgrades_daemonset" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "filebeat" "daemon" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} +{{ dict "envAll" $envAll "podName" "filebeat" "containerNames" (list "filebeat" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: + serviceAccountName: {{ $serviceAccountName }} +{{ if $envAll.Values.pod.tolerations.filebeat.enabled }} +{{ tuple $envAll "filebeat" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }} +{{ else }} + nodeSelector: + {{ .Values.labels.filebeat.node_selector_key }}: {{ .Values.labels.filebeat.node_selector_value | quote }} +{{ end }} + initContainers: +{{ tuple $envAll "filebeat" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: filebeat + image: {{ .Values.images.tags.filebeat }} + imagePullPolicy: {{ .Values.images.pull_policy }} + securityContext: + runAsUser: 0 +{{ tuple $envAll $envAll.Values.pod.resources.filebeat | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + args: + - "-e" + ports: + - name: filebeat + containerPort: {{ tuple "filebeat" "internal" "service" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: ELASTICSEARCH_HOST + value: {{ tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }} + - name: ELASTICSEARCH_PORT + value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: KIBANA_HOST + value: {{ tuple "kibana" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }} + - name: KIBANA_PORT + value: {{ tuple "kibana" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: ELASTICSEARCH_USERNAME + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_USERNAME + - name: ELASTICSEARCH_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_PASSWORD + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: data + mountPath: /usr/share/filebeat/data + - name: varlog + mountPath: /var/log + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + readOnly: true + - name: filebeat-etc + mountPath: /usr/share/filebeat/filebeat.yml + readOnly: true + subPath: filebeat.yml + - name: filebeat-etc + mountPath: /usr/share/filebeat/modules.d/system.yml + subPath: system.yml + readOnly: true +{{ if $mounts_filebeat.volumeMounts }}{{ toYaml $mounts_filebeat.volumeMounts | indent 8 }}{{ end }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: varlog + hostPath: + path: /var/log + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers + - name: filebeat-etc + configMap: + name: filebeat-etc + defaultMode: 0444 + - name: data + hostPath: + path: /var/lib/filebeat + type: DirectoryOrCreate +{{ if $mounts_filebeat.volumes }}{{ toYaml $mounts_filebeat.volumes | indent 8 }}{{ end }} +{{- end }} diff --git a/elastic-filebeat/templates/job-image-repo-sync.yaml b/elastic-filebeat/templates/job-image-repo-sync.yaml new file mode 100644 index 0000000000..8502f2b60b --- /dev/null +++ b/elastic-filebeat/templates/job-image-repo-sync.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "filebeat" -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/elastic-filebeat/templates/secret-elasticsearch-creds.yaml b/elastic-filebeat/templates/secret-elasticsearch-creds.yaml new file mode 100644 index 0000000000..347eaa9d0f --- /dev/null +++ b/elastic-filebeat/templates/secret-elasticsearch-creds.yaml @@ -0,0 +1,27 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_elasticsearch }} +{{- $envAll := . }} +{{- $secretName := index $envAll.Values.secrets.elasticsearch.user }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} +type: Opaque +data: + ELASTICSEARCH_USERNAME: {{ .Values.endpoints.elasticsearch.auth.admin.username | b64enc }} + ELASTICSEARCH_PASSWORD: {{ .Values.endpoints.elasticsearch.auth.admin.password | b64enc }} +{{- end }} diff --git a/elastic-filebeat/templates/secret-registry.yaml b/elastic-filebeat/templates/secret-registry.yaml new file mode 100644 index 0000000000..da979b3223 --- /dev/null +++ b/elastic-filebeat/templates/secret-registry.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_registry .Values.endpoints.oci_image_registry.auth.enabled }} +{{ include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) }} +{{- end }} diff --git a/elastic-filebeat/values.yaml b/elastic-filebeat/values.yaml new file mode 100644 index 0000000000..ca9111842f --- /dev/null +++ b/elastic-filebeat/values.yaml @@ -0,0 +1,287 @@ +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for filebeat +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +--- +release_group: null + +labels: + filebeat: + node_selector_key: openstack-control-plane + node_selector_value: enabled + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +images: + tags: + filebeat: docker.elastic.co/beats/filebeat-oss:7.1.0 + dep_check: quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal + image_repo_sync: docker.io/library/docker:17.07.0 + pull_policy: IfNotPresent + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +secrets: + elasticsearch: + user: filebeat-elasticsearch-user + oci_image_registry: + elastic-filebeat: elastic-filebeat-oci-image-registry-key + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - filebeat-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + filebeat: + services: + - endpoint: internal + service: kibana + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + +conf: + filebeat: + setup: + dashboards: + enabled: true + index: "filebeat-*" + retry: + enabled: true + interval: 5 + kibana: + host: "${KIBANA_HOST}:${KIBANA_PORT}" + username: "${ELASTICSEARCH_USERNAME}" + password: "${ELASTICSEARCH_PASSWORD}" + path: + logs: /var/log/ + output: + elasticsearch: + hosts: ["${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}/"] + username: "${ELASTICSEARCH_USERNAME}" + password: "${ELASTICSEARCH_PASSWORD}" + filebeat: + config: + modules: + path: ${path.config}/modules.d/*.yml + reload: + enabled: true + autodiscover: + providers: + - type: kubernetes + templates: + - condition: + equals: + kubernetes.namespace: kube-system + config: + - type: docker + containers.ids: + - "${data.kubernetes.container.id}" + exclude_lines: ["^\\s+[\\-`('.|_]"] + - type: kubernetes + templates: + - condition: + equals: + kubernetes.namespace: ceph + config: + - type: docker + containers.ids: + - "${data.kubernetes.container.id}" + exclude_lines: ["^\\s+[\\-`('.|_]"] + - type: kubernetes + templates: + - condition: + equals: + kubernetes.namespace: openstack + config: + - type: docker + containers.ids: + - "${data.kubernetes.container.id}" + exclude_lines: ["^\\s+[\\-`('.|_]"] + - type: kubernetes + templates: + - condition: + equals: + kubernetes.namespace: osh-infra + config: + - type: docker + containers.ids: + - "${data.kubernetes.container.id}" + exclude_lines: ["^\\s+[\\-`('.|_]"] + processors: + - add_kubernetes_metadata: + in_cluster: true + - drop_event: + when: + equals: + kubernetes: + container: + name: "filebeat" + modules: + system: + - module: system + syslog: + enabled: true + var.paths: ["/var/log/syslog*"] + fields: + host: + name: "${NODE_NAME}" + auth: + enabled: true + var.paths: ["/var/log/auth.log"] + fields: + host: + name: "${NODE_NAME}" + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + elastic-filebeat: + username: elastic-filebeat + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + elasticsearch: + namespace: null + name: elasticsearch + auth: + admin: + username: admin + password: changeme + hosts: + data: elasticsearch-data + default: elasticsearch-logging + discovery: elasticsearch-discovery + public: elasticsearch + host_fqdn_override: + default: null + path: + default: null + scheme: + default: http + port: + http: + default: 80 + kibana: + name: kibana + namespace: null + hosts: + default: kibana-dash + public: kibana + host_fqdn_override: + default: null + path: + default: null + scheme: + default: http + port: + kibana: + default: 5601 + http: + default: 80 + filebeat: + namespace: null + name: filebeat + hosts: + default: filebeat + host_fqdn_override: + default: null + path: + default: null + scheme: + default: http + port: + service: + default: 5066 + +pod: + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + weight: + default: 10 + lifecycle: + upgrades: + daemonsets: + pod_replacement_strategy: RollingUpdate + filebeat: + enabled: true + min_ready_seconds: 0 + max_unavailable: 1 + resources: + filebeat: + enabled: false + limits: + memory: '400Mi' + cpu: '400m' + requests: + memory: '100Mi' + cpu: '100m' + tolerations: + filebeat: + enabled: false + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + - key: node-role.kubernetes.io/control-plane + operator: Exists + - key: node-role.kubernetes.io/node + operator: Exists + mounts: + filebeat: + filebeat: + +manifests: + configmap_bin: true + configmap_etc: true + daemonset: true + job_image_repo_sync: true + secret_elasticsearch: true + secret_registry: true +... diff --git a/elastic-metricbeat/Chart.yaml b/elastic-metricbeat/Chart.yaml new file mode 100644 index 0000000000..a7562bbac6 --- /dev/null +++ b/elastic-metricbeat/Chart.yaml @@ -0,0 +1,29 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: v7.1.0 +description: OpenStack-Helm Elastic Metricbeat +name: elastic-metricbeat +version: 2024.2.0 +home: https://www.elastic.co/products/beats/metricbeat +sources: + - https://github.com/elastic/beats/tree/master/metricbeat + - https://opendev.org/openstack/openstack-helm-infra +maintainers: + - name: OpenStack-Helm Authors +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit/ + version: ">= 0.1.0" +... diff --git a/elastic-metricbeat/templates/configmap-etc.yaml b/elastic-metricbeat/templates/configmap-etc.yaml new file mode 100644 index 0000000000..322a2492c2 --- /dev/null +++ b/elastic-metricbeat/templates/configmap-etc.yaml @@ -0,0 +1,35 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_etc }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: metricbeat-etc +data: + metricbeat.yml: | +{{ toYaml .Values.conf.metricbeat | indent 4 }} + rabbitmq.yml: | +{{ toYaml .Values.conf.modules.rabbitmq | indent 4 }} + mysql.yml: | +{{ toYaml .Values.conf.modules.mysql | indent 4 }} + system.yml: | +{{ toYaml .Values.conf.modules.system | indent 4 }} + daemonset_kubernetes.yml: | +{{ toYaml .Values.conf.modules.daemonset_kubernetes | indent 4 }} + deployment_kubernetes.yml: | +{{ toYaml .Values.conf.modules.deployment_kubernetes | indent 4 }} +{{- end }} diff --git a/elastic-metricbeat/templates/daemonset-node-metrics.yaml b/elastic-metricbeat/templates/daemonset-node-metrics.yaml new file mode 100644 index 0000000000..e40e0c0961 --- /dev/null +++ b/elastic-metricbeat/templates/daemonset-node-metrics.yaml @@ -0,0 +1,176 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.daemonset }} +{{- $envAll := . }} +{{- $esUserSecret := .Values.secrets.elasticsearch.user }} + +{{- $mounts_metricbeat := .Values.pod.mounts.metricbeat.metricbeat }} + +{{- $serviceAccountName := printf "%s-%s" .Release.Name "metricbeat" }} +{{ tuple $envAll "metricbeat" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ $serviceAccountName }} + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ $serviceAccountName }} +rules: + - apiGroups: + - "" + resources: + - namespaces + - nodes + - pods + - services + - endpoints + - replicationcontrollers + - limitranges + - events + verbs: + - get + - list + - watch + - apiGroups: + - apps + resources: + - statefulsets + - daemonsets + - deployments + - replicasets + verbs: + - get + - list + - watch +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: metricbeat-node-modules +spec: + selector: + matchLabels: +{{ tuple $envAll "metricbeat" "daemon" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} +{{ tuple $envAll "metricbeat" | include "helm-toolkit.snippets.kubernetes_upgrades_daemonset" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "metricbeat" "daemon" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} + spec: + hostNetwork: true + dnsPolicy: {{ .Values.pod.dns_policy }} + serviceAccountName: {{ $serviceAccountName }} +{{ if $envAll.Values.pod.tolerations.metricbeat.enabled }} +{{ tuple $envAll "metricbeat" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }} +{{ else }} + nodeSelector: + {{ .Values.labels.metricbeat.node_selector_key }}: {{ .Values.labels.metricbeat.node_selector_value | quote }} +{{ end }} + initContainers: +{{ tuple $envAll "metricbeat" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: metricbeat + securityContext: + privileged: true + runAsUser: 0 + image: {{ .Values.images.tags.metricbeat }} + imagePullPolicy: {{ .Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.pod.resources.metricbeat | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + args: + - "-c" + - "/usr/share/metricbeat/metricbeat.yml" + - "-e" + - "-system.hostfs=/hostfs" + env: + - name: ELASTICSEARCH_HOST + value: {{ tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }} + - name: ELASTICSEARCH_PORT + value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: KIBANA_HOST + value: {{ tuple "kibana" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }} + - name: KIBANA_PORT + value: {{ tuple "kibana" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: ELASTICSEARCH_USERNAME + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_USERNAME + - name: ELASTICSEARCH_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_PASSWORD + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: metricbeat-etc + mountPath: /usr/share/metricbeat/metricbeat.yml + subPath: metricbeat.yml + readOnly: true + - name: metricbeat-etc + mountPath: /usr/share/metricbeat/modules.d/system.yml + subPath: system.yml + readOnly: true + - name: metricbeat-etc + mountPath: /usr/share/metricbeat/modules.d/kubernetes.yml + subPath: daemonset_kubernetes.yml + readOnly: true + - name: dockersock + mountPath: /var/run/docker.sock + - name: proc + mountPath: /hostfs/proc + readOnly: true + - name: cgroup + mountPath: /hostfs/sys/fs/cgroup + readOnly: true +{{ if $mounts_metricbeat.volumeMounts }}{{ toYaml $mounts_metricbeat.volumeMounts | indent 12 }}{{ end }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: proc + hostPath: + path: /proc + - name: cgroup + hostPath: + path: /sys/fs/cgroup + - name: dockersock + hostPath: + path: /var/run/docker.sock + - name: metricbeat-etc + configMap: + defaultMode: 0444 + name: metricbeat-etc + - name: data + emptyDir: {} +{{ if $mounts_metricbeat.volumes }}{{ toYaml $mounts_metricbeat.volumes | indent 8 }}{{ end }} +{{- end }} diff --git a/elastic-metricbeat/templates/deployment-modules.yaml b/elastic-metricbeat/templates/deployment-modules.yaml new file mode 100644 index 0000000000..e784cdd19b --- /dev/null +++ b/elastic-metricbeat/templates/deployment-modules.yaml @@ -0,0 +1,158 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.deployment }} +{{- $envAll := . }} + +{{- $esUserSecret := .Values.secrets.elasticsearch.user }} + +{{- $serviceAccountName := printf "%s-%s" .Release.Name "metricbeat-deployments" }} +{{ tuple $envAll "metricbeat" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ $serviceAccountName }} + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ $serviceAccountName }} +rules: + - apiGroups: + - "" + resources: + - namespaces + - nodes + - pods + - services + - endpoints + - replicationcontrollers + - limitranges + - events + verbs: + - get + - list + - watch + - apiGroups: + - apps + resources: + - statefulsets + - daemonsets + - deployments + - replicasets + verbs: + - get + - list + - watch +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: metricbeat-deployment-modules + labels: +{{ tuple $envAll "metricbeat" "deployment-modules" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + replicas: {{ .Values.pod.replicas.metricbeat }} + selector: + matchLabels: +{{ tuple $envAll "metricbeat" "deployment-modules" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} +{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "metricbeat" "deployment-modules" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} + spec: + serviceAccountName: {{ $serviceAccountName }} + affinity: +{{ tuple $envAll "metricbeat" "deployment-modules" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} + nodeSelector: + {{ .Values.labels.metricbeat.node_selector_key }}: {{ .Values.labels.metricbeat.node_selector_value }} + initContainers: +{{ tuple $envAll "metricbeat" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: metricbeat + securityContext: + runAsUser: 0 +{{ tuple $envAll "metricbeat" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.metricbeat | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + args: + - "-c" + - "/usr/share/metricbeat/metricbeat.yml" + - "-e" + env: + - name: ELASTICSEARCH_HOST + value: {{ tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }} + - name: ELASTICSEARCH_PORT + value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: KUBE_STATE_METRICS_HOST + value: {{ tuple "kube_state_metrics" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }} + - name: KUBE_STATE_METRICS_PORT + value: {{ tuple "kube_state_metrics" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: KIBANA_HOST + value: {{ tuple "kibana" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }} + - name: KIBANA_PORT + value: {{ tuple "kibana" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: ELASTICSEARCH_USERNAME + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_USERNAME + - name: ELASTICSEARCH_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_PASSWORD + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: metricbeat-etc + mountPath: /usr/share/metricbeat/metricbeat.yml + subPath: metricbeat.yml + readOnly: true + - name: metricbeat-etc + mountPath: /usr/share/metricbeat/modules.d/kubernetes.yml + subPath: deployment_kubernetes.yml + readOnly: true + - name: metricbeat-etc + mountPath: /usr/share/metricbeat/modules.d/mysql.yml + subPath: mysql.yml + readOnly: true + - name: metricbeat-etc + mountPath: /usr/share/metricbeat/modules.d/rabbitmq.yml + subPath: rabbitmq.yml + readOnly: true + volumes: + - name: pod-tmp + emptyDir: {} + - name: metricbeat-etc + configMap: + name: metricbeat-etc + defaultMode: 0444 +{{- end }} diff --git a/elastic-metricbeat/templates/job-image-repo-sync.yaml b/elastic-metricbeat/templates/job-image-repo-sync.yaml new file mode 100644 index 0000000000..bcff2baf88 --- /dev/null +++ b/elastic-metricbeat/templates/job-image-repo-sync.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "metricbeat" -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/elastic-metricbeat/templates/secret-elasticsearch-creds.yaml b/elastic-metricbeat/templates/secret-elasticsearch-creds.yaml new file mode 100644 index 0000000000..347eaa9d0f --- /dev/null +++ b/elastic-metricbeat/templates/secret-elasticsearch-creds.yaml @@ -0,0 +1,27 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_elasticsearch }} +{{- $envAll := . }} +{{- $secretName := index $envAll.Values.secrets.elasticsearch.user }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} +type: Opaque +data: + ELASTICSEARCH_USERNAME: {{ .Values.endpoints.elasticsearch.auth.admin.username | b64enc }} + ELASTICSEARCH_PASSWORD: {{ .Values.endpoints.elasticsearch.auth.admin.password | b64enc }} +{{- end }} diff --git a/elastic-metricbeat/templates/secret-registry.yaml b/elastic-metricbeat/templates/secret-registry.yaml new file mode 100644 index 0000000000..da979b3223 --- /dev/null +++ b/elastic-metricbeat/templates/secret-registry.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_registry .Values.endpoints.oci_image_registry.auth.enabled }} +{{ include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) }} +{{- end }} diff --git a/elastic-metricbeat/values.yaml b/elastic-metricbeat/values.yaml new file mode 100644 index 0000000000..ff083b5792 --- /dev/null +++ b/elastic-metricbeat/values.yaml @@ -0,0 +1,286 @@ +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for metricbeat +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +--- +release_group: null + +labels: + metricbeat: + node_selector_key: openstack-control-plane + node_selector_value: enabled + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +images: + tags: + metricbeat: docker.elastic.co/beats/metricbeat-oss:7.1.0 + dep_check: quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal + image_repo_sync: docker.io/library/docker:17.07.0 + pull_policy: IfNotPresent + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +secrets: + elasticsearch: + user: metricbeat-elasticsearch-user + oci_image_registry: + elastic-metricbeat: elastic-metricbeat-oci-image-registry-key + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - metricbeat-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + metricbeat: + services: + - endpoint: internal + service: kibana + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + +conf: + metricbeat: + setup: + dashboards: + enabled: true + index: metricbeat-* + retry: + enabled: true + interval: 5 + kibana: + host: "${KIBANA_HOST}:${KIBANA_PORT}" + username: "${ELASTICSEARCH_USERNAME}" + password: "${ELASTICSEARCH_PASSWORD}" + metricbeat: + config: + modules: + path: ${path.config}/modules.d/*.yml + reload: + enabled: true + output: + elasticsearch: + hosts: ['${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}'] + username: ${ELASTICSEARCH_USERNAME} + password: ${ELASTICSEARCH_PASSWORD} + modules: + docker: + - module: docker + metricsets: + - "container" + - "cpu" + - "diskio" + - "healthcheck" + - "info" + - "image" + - "memory" + - "network" + hosts: ["unix:///var/run/docker.sock"] + period: 10s + enabled: true + system: + - module: system + period: 10s + metricsets: + - cpu + - load + - memory + - network + - process + - process_summary + - core + - diskio + - socket + - filesystem + - fsstat + processes: ['.*'] + cpu.metrics: ["percentages"] + core.metrics: ["percentages"] + process.include_top_n: + by_cpu: 5 + by_memory: 5 + enabled: true + daemonset_kubernetes: + - module: kubernetes + metricsets: + - node + - system + - pod + - container + - volume + period: 10s + hosts: ["localhost:10255"] + add_metadata: true + in_cluster: true + enabled: true + deployment_kubernetes: + - module: kubernetes + metricsets: + - state_node + - state_deployment + - state_replicaset + - state_pod + - state_container + - event + period: 10s + hosts: ['${KUBE_STATE_METRICS_HOST}:${KUBE_STATE_METRICS_PORT}'] + add_metadata: true + in_cluster: true + enabled: true + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + elastic-metricbeat: + username: elastic-metricbeat + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + kube_state_metrics: + namespace: null + hosts: + default: kube-state-metrics + host_fqdn_override: + default: null + path: + default: null + scheme: + default: 'http' + port: + metrics: + default: 8080 + elasticsearch: + namespace: null + name: elasticsearch + auth: + admin: + username: admin + password: changeme + hosts: + data: elasticsearch-data + default: elasticsearch-logging + discovery: elasticsearch-discovery + public: elasticsearch + host_fqdn_override: + default: null + path: + default: null + scheme: + default: http + port: + http: + default: 80 + kibana: + name: kibana + namespace: osh-infra + hosts: + default: kibana-dash + public: kibana + host_fqdn_override: + default: null + path: + default: null + scheme: + default: http + port: + kibana: + default: 5601 + http: + default: 80 + +pod: + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + weight: + default: 10 + lifecycle: + upgrades: + daemonsets: + pod_replacement_strategy: RollingUpdate + metricbeat: + enabled: true + min_ready_seconds: 0 + max_unavailable: 1 + dns_policy: "ClusterFirstWithHostNet" + replicas: + metricbeat: 1 + resources: + metricbeat: + enabled: false + limits: + memory: '400Mi' + cpu: '400m' + requests: + memory: '100Mi' + cpu: '100m' + tolerations: + metricbeat: + enabled: false + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + - key: node-role.kubernetes.io/control-plane + operator: Exists + - key: node-role.kubernetes.io/node + operator: Exists + mounts: + metricbeat: + metricbeat: + +manifests: + configmap_bin: true + configmap_etc: true + daemonset: true + deployment: true + job_image_repo_sync: true + secret_elasticsearch: true + secret_registry: true +... diff --git a/elastic-packetbeat/Chart.yaml b/elastic-packetbeat/Chart.yaml new file mode 100644 index 0000000000..909032785a --- /dev/null +++ b/elastic-packetbeat/Chart.yaml @@ -0,0 +1,29 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: v7.1.0 +description: OpenStack-Helm Elastic Packetbeat +name: elastic-packetbeat +version: 2024.2.0 +home: https://www.elastic.co/products/beats/packetbeat +sources: + - https://github.com/elastic/beats/tree/master/packetbeat + - https://opendev.org/openstack/openstack-helm-infra +maintainers: + - name: OpenStack-Helm Authors +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit/ + version: ">= 0.1.0" +... diff --git a/elastic-packetbeat/templates/configmap-etc.yaml b/elastic-packetbeat/templates/configmap-etc.yaml new file mode 100644 index 0000000000..359bd13d0e --- /dev/null +++ b/elastic-packetbeat/templates/configmap-etc.yaml @@ -0,0 +1,25 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_etc }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: packetbeat-etc +data: + packetbeat.yml: | +{{ toYaml .Values.conf.packetbeat | indent 4 }} +{{- end }} diff --git a/elastic-packetbeat/templates/daemonset.yaml b/elastic-packetbeat/templates/daemonset.yaml new file mode 100644 index 0000000000..486cc7fe0e --- /dev/null +++ b/elastic-packetbeat/templates/daemonset.yaml @@ -0,0 +1,145 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.daemonset }} +{{- $envAll := . }} +{{- $esUserSecret := .Values.secrets.elasticsearch.user }} + +{{- $mounts_packetbeat := .Values.pod.mounts.packetbeat.packetbeat }} + +{{- $serviceAccountName := printf "%s-%s" .Release.Name "packetbeat" }} +{{ tuple $envAll "packetbeat" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ $serviceAccountName }} + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ $serviceAccountName }} +rules: + - apiGroups: + - "" + resources: + - namespaces + - nodes + - pods + - services + - endpoints + - replicationcontrollers + - limitranges + verbs: + - get + - list + - watch + - apiGroups: + - apps + resources: + - statefulsets + - daemonsets + - deployments + - replicasets + verbs: + - get + - list + - watch +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: packetbeat +spec: + selector: + matchLabels: +{{ tuple $envAll "packetbeat" "daemon" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} +{{ tuple $envAll "packetbeat" | include "helm-toolkit.snippets.kubernetes_upgrades_daemonset" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "packetbeat" "daemon" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} + spec: + securityContext: + runAsUser: 0 + hostNetwork: true + dnsPolicy: {{ .Values.pod.dns_policy }} + serviceAccountName: {{ $serviceAccountName }} + initContainers: +{{ tuple $envAll "packetbeat" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: packetbeat + image: {{ .Values.images.tags.packetbeat }} + imagePullPolicy: {{ .Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.pod.resources.packetbeat | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + securityContext: + privileged: true + capabilities: + add: + - NET_ADMIN + args: + - "-c" + - "/usr/share/packetbeat/packetbeat.yml" + - "-e" + env: + - name: ELASTICSEARCH_HOST + value: {{ tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }} + - name: ELASTICSEARCH_PORT + value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: KIBANA_HOST + value: {{ tuple "kibana" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | quote }} + - name: KIBANA_PORT + value: {{ tuple "kibana" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: ELASTICSEARCH_USERNAME + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_USERNAME + - name: ELASTICSEARCH_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_PASSWORD + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: packetbeat-etc + mountPath: /usr/share/packetbeat/packetbeat.yml + subPath: packetbeat.yml + readOnly: true +{{ if $mounts_packetbeat.volumeMounts }}{{ toYaml $mounts_packetbeat.volumeMounts | indent 12 }}{{ end }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: packetbeat-etc + configMap: + defaultMode: 0444 + name: packetbeat-etc +{{ if $mounts_packetbeat.volumes }}{{ toYaml $mounts_packetbeat.volumes | indent 8 }}{{ end }} +{{- end }} diff --git a/elastic-packetbeat/templates/job-image-repo-sync.yaml b/elastic-packetbeat/templates/job-image-repo-sync.yaml new file mode 100644 index 0000000000..bcff2baf88 --- /dev/null +++ b/elastic-packetbeat/templates/job-image-repo-sync.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "metricbeat" -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/elastic-packetbeat/templates/secret-elasticsearch-creds.yaml b/elastic-packetbeat/templates/secret-elasticsearch-creds.yaml new file mode 100644 index 0000000000..347eaa9d0f --- /dev/null +++ b/elastic-packetbeat/templates/secret-elasticsearch-creds.yaml @@ -0,0 +1,27 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_elasticsearch }} +{{- $envAll := . }} +{{- $secretName := index $envAll.Values.secrets.elasticsearch.user }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} +type: Opaque +data: + ELASTICSEARCH_USERNAME: {{ .Values.endpoints.elasticsearch.auth.admin.username | b64enc }} + ELASTICSEARCH_PASSWORD: {{ .Values.endpoints.elasticsearch.auth.admin.password | b64enc }} +{{- end }} diff --git a/elastic-packetbeat/templates/secret-registry.yaml b/elastic-packetbeat/templates/secret-registry.yaml new file mode 100644 index 0000000000..da979b3223 --- /dev/null +++ b/elastic-packetbeat/templates/secret-registry.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_registry .Values.endpoints.oci_image_registry.auth.enabled }} +{{ include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) }} +{{- end }} diff --git a/elastic-packetbeat/values.yaml b/elastic-packetbeat/values.yaml new file mode 100644 index 0000000000..168a19acb6 --- /dev/null +++ b/elastic-packetbeat/values.yaml @@ -0,0 +1,203 @@ +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for packetbeat +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +--- +release_group: null + +labels: + packetbeat: + node_selector_key: openstack-control-plane + node_selector_value: enabled + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +images: + tags: + packetbeat: docker.elastic.co/beats/packetbeat-oss:7.1.0 + dep_check: quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal + image_repo_sync: docker.io/library/docker:17.07.0 + pull_policy: IfNotPresent + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +secrets: + elasticsearch: + user: packetbeat-elasticsearch-user + oci_image_registry: + elastic-packetbeat: elastic-packetbeat-oci-image-registry-key + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - packetbeat-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + packetbeat: + services: null + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + +conf: + packetbeat: + setup: + kibana: + host: "${KIBANA_HOST}:${KIBANA_PORT}" + username: "${ELASTICSEARCH_USERNAME}" + password: "${ELASTICSEARCH_PASSWORD}" + dashboards: + enabled: true + index: "packetbeat-*" + retry: + enabled: true + interval: 5 + packetbeat: + flows: + timeout: 30s + period: 10s + interfaces: + device: any + protocols: + - type: dhcpv4 + ports: [67, 68] + - type: dns + ports: [53] + include_authorities: true + include_additionals: true + - type: http + ports: [80, 8080, 8081, 5000, 8002, 6666, 3000, 5601, 9100, 9090, 44134] + output: + elasticsearch: + hosts: ['${ELASTICSEARCH_HOST}:${ELASTICSEARCH_PORT}'] + username: ${ELASTICSEARCH_USERNAME} + password: ${ELASTICSEARCH_PASSWORD} + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + elastic-packetbeat: + username: elastic-packetbeat + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + elasticsearch: + name: elasticsearch + namespace: null + auth: + admin: + username: admin + password: changeme + hosts: + data: elasticsearch-data + default: elasticsearch-logging + discovery: elasticsearch-discovery + public: elasticsearch + host_fqdn_override: + default: null + path: + default: null + scheme: + default: http + port: + http: + default: 80 + kibana: + name: kibana + namespace: null + hosts: + default: kibana-dash + public: kibana + host_fqdn_override: + default: null + path: + default: null + scheme: + default: http + port: + kibana: + default: 5601 + http: + default: 80 + +pod: + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + lifecycle: + upgrades: + daemonsets: + pod_replacement_strategy: RollingUpdate + packetbeat: + enabled: true + min_ready_seconds: 0 + max_unavailable: 1 + dns_policy: "ClusterFirstWithHostNet" + replicas: + packetbeat: 1 + resources: + packetbeat: + enabled: false + limits: + memory: '400Mi' + cpu: '400m' + requests: + memory: '100Mi' + cpu: '100m' + mounts: + packetbeat: + packetbeat: + +manifests: + configmap_bin: true + configmap_etc: true + daemonset: true + job_image_repo_sync: true + secret_elasticsearch: true + secret_registry: true +... diff --git a/elasticsearch/Chart.yaml b/elasticsearch/Chart.yaml new file mode 100644 index 0000000000..ab9bcf7712 --- /dev/null +++ b/elasticsearch/Chart.yaml @@ -0,0 +1,29 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: v8.9.0 +description: OpenStack-Helm ElasticSearch +name: elasticsearch +version: 2024.2.0 +home: https://www.elastic.co/ +sources: + - https://github.com/elastic/elasticsearch + - https://opendev.org/openstack/openstack-helm-addons +maintainers: + - name: OpenStack-Helm Authors +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit + version: ">= 0.1.0" +... diff --git a/elasticsearch/templates/bin/_apache.sh.tpl b/elasticsearch/templates/bin/_apache.sh.tpl new file mode 100644 index 0000000000..1032028cc6 --- /dev/null +++ b/elasticsearch/templates/bin/_apache.sh.tpl @@ -0,0 +1,48 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ev + +COMMAND="${@:-start}" + +function start () { + + if [ -f /etc/apache2/envvars ]; then + # Loading Apache2 ENV variables + source /etc/httpd/apache2/envvars + fi + # Apache gets grumpy about PID files pre-existing + rm -f /etc/httpd/logs/httpd.pid + + if [ -f /usr/local/apache2/conf/.htpasswd ]; then + htpasswd -b /usr/local/apache2/conf/.htpasswd "$ELASTICSEARCH_USERNAME" "$ELASTICSEARCH_PASSWORD" + else + htpasswd -cb /usr/local/apache2/conf/.htpasswd "$ELASTICSEARCH_USERNAME" "$ELASTICSEARCH_PASSWORD" + fi + + if [ ! -z $ELASTICSEARCH_LOGGING_USERNAME ]; then + htpasswd -b /usr/local/apache2/conf/.htpasswd "$ELASTICSEARCH_LOGGING_USERNAME" "$ELASTICSEARCH_LOGGING_PASSWORD" + fi + + #Launch Apache on Foreground + exec httpd -DFOREGROUND +} + +function stop () { + apachectl -k graceful-stop +} + +$COMMAND diff --git a/elasticsearch/templates/bin/_ceph-admin-keyring.sh.tpl b/elasticsearch/templates/bin/_ceph-admin-keyring.sh.tpl new file mode 100644 index 0000000000..f19bf03e05 --- /dev/null +++ b/elasticsearch/templates/bin/_ceph-admin-keyring.sh.tpl @@ -0,0 +1,29 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +export HOME=/tmp + +cat < /etc/ceph/ceph.client.admin.keyring +[client.admin] +{{- if .Values.conf.ceph.admin_keyring }} + key = {{ .Values.conf.ceph.admin_keyring }} +{{- else }} + key = $(cat /tmp/client-keyring) +{{- end }} +EOF + +exit 0 diff --git a/elasticsearch/templates/bin/_create_s3_buckets.sh.tpl b/elasticsearch/templates/bin/_create_s3_buckets.sh.tpl new file mode 100644 index 0000000000..c21df06613 --- /dev/null +++ b/elasticsearch/templates/bin/_create_s3_buckets.sh.tpl @@ -0,0 +1,66 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +#!/bin/bash + +set -e + +function check_rgw_s3_bucket () { + echo "Checking if bucket exists" + s3cmd $CONNECTION_ARGS $USER_AUTH_ARGS ls s3://$S3_BUCKET +} + +function create_rgw_s3_bucket () { + echo "Creating bucket" + s3cmd $CONNECTION_ARGS $S3_BUCKET_OPTS $USER_AUTH_ARGS mb s3://$S3_BUCKET +} + +function modify_bucket_acl () { + echo "Updating bucket ACL" + s3cmd $CONNECTION_ARGS $USER_AUTH_ARGS setacl s3://$S3_BUCKET --acl-grant=read:$S3_USERNAME --acl-grant=write:$S3_USERNAME +} + +ADMIN_AUTH_ARGS=" --access_key=$S3_ADMIN_ACCESS_KEY --secret_key=$S3_ADMIN_SECRET_KEY" + +{{- $envAll := . }} +{{- range $bucket := .Values.storage.s3.buckets }} + +S3_BUCKET={{ $bucket.name }} +S3_BUCKET_OPTS={{ $bucket.options | default nil | include "helm-toolkit.utils.joinListWithSpace" }} +S3_SSL_OPT={{ $bucket.ssl_connection_option | default "" }} + +S3_USERNAME=${{ printf "%s_S3_USERNAME" ( $bucket.client | replace "-" "_" | upper) }} +S3_ACCESS_KEY=${{ printf "%s_S3_ACCESS_KEY" ( $bucket.client | replace "-" "_" | upper) }} +S3_SECRET_KEY=${{ printf "%s_S3_SECRET_KEY" ( $bucket.client | replace "-" "_" | upper) }} + +{{- with $client := index $envAll.Values.storage.s3.clients $bucket.client }} + +RGW_HOST={{ $client.settings.endpoint | default (tuple "ceph_object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup") }} +RGW_PROTO={{ $client.settings.protocol | default (tuple "ceph_object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup") }} + +{{- end }} + +CONNECTION_ARGS="--host=$RGW_HOST --host-bucket=$RGW_HOST" +if [ "$RGW_PROTO" = "http" ]; then + CONNECTION_ARGS+=" --no-ssl" +else + CONNECTION_ARGS+=" $S3_SSL_OPT" +fi + +USER_AUTH_ARGS=" --access_key=$S3_ACCESS_KEY --secret_key=$S3_SECRET_KEY" + +echo "Creating Bucket $S3_BUCKET at $RGW_HOST" +check_rgw_s3_bucket || ( create_rgw_s3_bucket && modify_bucket_acl ) + +{{- end }} diff --git a/elasticsearch/templates/bin/_create_s3_users.sh.tpl b/elasticsearch/templates/bin/_create_s3_users.sh.tpl new file mode 100644 index 0000000000..1d3962317f --- /dev/null +++ b/elasticsearch/templates/bin/_create_s3_users.sh.tpl @@ -0,0 +1,75 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +#!/bin/bash + +set -e + +function create_s3_user () { + echo "Creating s3 user and key pair" + radosgw-admin user create \ + --uid=${S3_USERNAME} \ + --display-name=${S3_USERNAME} \ + --key-type=s3 \ + --access-key ${S3_ACCESS_KEY} \ + --secret-key ${S3_SECRET_KEY} +} + +function update_s3_user () { + # Retrieve old access keys, if they exist + old_access_keys=$(radosgw-admin user info --uid=${S3_USERNAME} \ + | jq -r '.keys[].access_key' || true) + if [[ ! -z ${old_access_keys} ]]; then + for access_key in $old_access_keys; do + # If current access key is the same as the key supplied, do nothing. + if [ "$access_key" == "${S3_ACCESS_KEY}" ]; then + echo "Current user and key pair exists." + continue + else + # If keys differ, remove previous key + radosgw-admin key rm --uid=${S3_USERNAME} --key-type=s3 --access-key=$access_key + fi + done + fi + # Perform one more additional check to account for scenarios where multiple + # key pairs existed previously, but one existing key was the supplied key + current_access_key=$(radosgw-admin user info --uid=${S3_USERNAME} \ + | jq -r '.keys[].access_key' || true) + # If the supplied key does not exist, modify the user + if [[ -z ${current_access_key} ]]; then + # Modify user with new access and secret keys + echo "Updating existing user's key pair" + radosgw-admin user modify \ + --uid=${S3_USERNAME}\ + --access-key ${S3_ACCESS_KEY} \ + --secret-key ${S3_SECRET_KEY} + fi +} + +{{- range $client, $config := .Values.storage.s3.clients -}} +{{- if $config.create_user | default false }} + +S3_USERNAME=${{ printf "%s_S3_USERNAME" ($client | replace "-" "_" | upper) }} +S3_ACCESS_KEY=${{ printf "%s_S3_ACCESS_KEY" ($client | replace "-" "_" | upper) }} +S3_SECRET_KEY=${{ printf "%s_S3_SECRET_KEY" ($client | replace "-" "_" | upper) }} + +user_exists=$(radosgw-admin user info --uid=${S3_USERNAME} || true) +if [[ -z ${user_exists} ]]; then + echo "Creating $S3_USERNAME" + create_s3_user > /dev/null 2>&1 +else + echo "Updating $S3_USERNAME" + update_s3_user > /dev/null 2>&1 +fi + +{{- end }} +{{- end }} diff --git a/elasticsearch/templates/bin/_create_template.sh.tpl b/elasticsearch/templates/bin/_create_template.sh.tpl new file mode 100644 index 0000000000..aee2674c54 --- /dev/null +++ b/elasticsearch/templates/bin/_create_template.sh.tpl @@ -0,0 +1,43 @@ +#!/bin/bash +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -e + +NUM_ERRORS=0 + +{{ range $name, $object := .Values.conf.api_objects }} +{{ if not (empty $object) }} + +echo "creating {{$name}}" +error=$(curl ${CACERT_OPTION} -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ + -X{{ $object.method | default "PUT" | upper }} \ + "${ELASTICSEARCH_ENDPOINT}/{{ $object.endpoint }}" \ + -H 'Content-Type: application/json' -d '{{ $object.body | toJson }}' | jq -r '.error') + +if [ $error == "null" ]; then + echo "Object {{$name}} was created." +else + echo "Error when creating object {{$name}}: $(echo $error | jq -r)" + NUM_ERRORS=$(($NUM_ERRORS+1)) +fi + +{{ end }} +{{ end }} + +if [ $NUM_ERRORS -gt 0 ]; then + exit 1 +else + echo "leaving normally" +fi diff --git a/elasticsearch/templates/bin/_curator.sh.tpl b/elasticsearch/templates/bin/_curator.sh.tpl new file mode 100644 index 0000000000..c2a5fa0b8e --- /dev/null +++ b/elasticsearch/templates/bin/_curator.sh.tpl @@ -0,0 +1,18 @@ +#!/bin/sh +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +exec {{ .Values.conf.curator.executable }} --config /etc/config/config.yml /etc/config/action_file.yml diff --git a/elasticsearch/templates/bin/_elasticsearch.sh.tpl b/elasticsearch/templates/bin/_elasticsearch.sh.tpl new file mode 100644 index 0000000000..93abde3d71 --- /dev/null +++ b/elasticsearch/templates/bin/_elasticsearch.sh.tpl @@ -0,0 +1,150 @@ +#!/bin/bash +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- $envAll := . }} + +set -e +COMMAND="${@:-start}" + +function initiate_keystore () { + elasticsearch-keystore create + {{- if .Values.conf.elasticsearch.snapshots.enabled }} + {{- range $client, $settings := .Values.storage.s3.clients -}} + {{- $access_key := printf "%s_S3_ACCESS_KEY" ( $client | replace "-" "_" | upper) }} + {{- $secret_key := printf "%s_S3_SECRET_KEY" ( $client | replace "-" "_" | upper) }} + echo ${{$access_key}} | elasticsearch-keystore add -xf s3.client.{{ $client }}.access_key + echo ${{$secret_key}} | elasticsearch-keystore add -xf s3.client.{{ $client }}.secret_key + {{- end }} + {{- end }} + + {{- if .Values.manifests.certificates }} + {{- $alias := .Values.secrets.tls.elasticsearch.elasticsearch.internal }} + JAVA_KEYTOOL_PATH=/usr/share/elasticsearch/jdk/bin/keytool + TRUSTSTORE_PATH=/usr/share/elasticsearch/config/elasticsearch-java-truststore + ${JAVA_KEYTOOL_PATH} -importcert -alias {{$alias}} -keystore ${TRUSTSTORE_PATH} -trustcacerts -noprompt -file ${JAVA_KEYSTORE_CERT_PATH} -storepass ${ELASTICSEARCH_PASSWORD} + ${JAVA_KEYTOOL_PATH} -storepasswd -keystore ${TRUSTSTORE_PATH} -new ${ELASTICSEARCH_PASSWORD} -storepass ${ELASTICSEARCH_PASSWORD} + {{- end }} +} + +function start () { + initiate_keystore + exec /usr/local/bin/docker-entrypoint.sh elasticsearch +} + +function stop () { + kill -TERM 1 +} + +function wait_to_join() { + # delay 5 seconds before the first check + sleep 5 + joined=$(curl -s ${CACERT_OPTION} -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" "${ELASTICSEARCH_ENDPOINT}/_cat/nodes" | grep -w $NODE_NAME || true ) + i=0 + while [ -z "$joined" ]; do + sleep 5 + joined=$(curl -s ${CACERT_OPTION} -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" "${ELASTICSEARCH_ENDPOINT}/_cat/nodes" | grep -w $NODE_NAME || true ) + i=$((i+1)) + # Waiting for up to 60 minutes + if [ $i -gt 720 ]; then + break + fi + done +} + +function allocate_data_node () { + echo "Node ${NODE_NAME} has started. Waiting to rejoin the cluster." + wait_to_join + echo "Re-enabling Replica Shard Allocation" + curl -s ${CACERT_OPTION} -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" -XPUT -H 'Content-Type: application/json' \ + "${ELASTICSEARCH_ENDPOINT}/_cluster/settings" -d "{ + \"persistent\": { + \"cluster.routing.allocation.enable\": null + } + }" +} + +function start_master_node () { + initiate_keystore + if [ ! -f {{ $envAll.Values.conf.elasticsearch.config.path.data }}/cluster-bootstrap.txt ]; + then + {{ if empty $envAll.Values.conf.elasticsearch.config.cluster.initial_master_nodes -}} + {{- $_ := set $envAll.Values "__eligible_masters" ( list ) }} + {{- range $podInt := until ( atoi (print $envAll.Values.pod.replicas.master ) ) }} + {{- $eligibleMaster := printf "elasticsearch-master-%s" (toString $podInt) }} + {{- $__eligible_masters := append $envAll.Values.__eligible_masters $eligibleMaster }} + {{- $_ := set $envAll.Values "__eligible_masters" $__eligible_masters }} + {{- end -}} + {{- $masters := include "helm-toolkit.utils.joinListWithComma" $envAll.Values.__eligible_masters -}} + echo {{$masters}} >> {{ $envAll.Values.conf.elasticsearch.config.path.data }}/cluster-bootstrap.txt + exec /usr/local/bin/docker-entrypoint.sh elasticsearch -Ecluster.initial_master_nodes={{$masters}} + {{- end }} + else + exec /usr/local/bin/docker-entrypoint.sh elasticsearch + fi +} + +function start_data_node () { + initiate_keystore + allocate_data_node & + /usr/local/bin/docker-entrypoint.sh elasticsearch & + function drain_data_node () { + + # Implement the Rolling Restart Protocol Described Here: + # https://www.elastic.co/guide/en/elasticsearch/reference/7.x/restart-cluster.html#restart-cluster-rolling + + echo "Disabling Replica Shard Allocation" + curl -s ${CACERT_OPTION} -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" -XPUT -H 'Content-Type: application/json' \ + "${ELASTICSEARCH_ENDPOINT}/_cluster/settings" -d "{ + \"persistent\": { + \"cluster.routing.allocation.enable\": \"primaries\" + } + }" + + # If version < 7.6 use _flush/synced; otherwise use _flush + # https://www.elastic.co/guide/en/elasticsearch/reference/current/indices-synced-flush-api.html#indices-synced-flush-api + + version=$(curl -s ${CACERT_OPTION} -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" "${ELASTICSEARCH_ENDPOINT}/" | jq -r .version.number) + + if [[ $version =~ "7.1" ]]; then + action="_flush/synced" + else + action="_flush" + fi + + curl -s ${CACERT_OPTION} -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" -XPOST "${ELASTICSEARCH_ENDPOINT}/$action" + + # TODO: Check the response of synced flush operations to make sure there are no failures. + # Synced flush operations that fail due to pending indexing operations are listed in the response body, + # although the request itself still returns a 200 OK status. If there are failures, reissue the request. + # (The only side effect of not doing so is slower start up times. See flush documentation linked above) + + echo "Node ${NODE_NAME} is ready to shutdown" + + echo "Killing Elasticsearch background processes" + jobs -p | xargs -t -r kill -TERM + wait + + # remove the trap handler + trap - TERM EXIT HUP INT + + echo "Node ${NODE_NAME} shutdown is complete" + exit 0 + } + trap drain_data_node TERM EXIT HUP INT + wait + +} + +$COMMAND diff --git a/elasticsearch/templates/bin/_helm-tests.sh.tpl b/elasticsearch/templates/bin/_helm-tests.sh.tpl new file mode 100644 index 0000000000..c9891512ed --- /dev/null +++ b/elasticsearch/templates/bin/_helm-tests.sh.tpl @@ -0,0 +1,47 @@ +#!/bin/bash +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +function create_test_index () { + index_result=$(curl ${CACERT_OPTION} -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ + -XPUT "${ELASTICSEARCH_ENDPOINT}/test_index?pretty" -H 'Content-Type: application/json' -d' + { + "settings" : { + "index" : { + "number_of_shards" : 3, + "number_of_replicas" : 2 + } + } + } + ' | grep -o '"acknowledged" *: *true') + + if [ -n "$index_result" ]; then + echo "PASS: Test index created!"; + else + echo "FAIL: Test index not created!"; + exit 1; + fi +} + +function remove_test_index () { + echo "Deleting index created for service testing" + curl ${CACERT_OPTION} -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ + -XDELETE "${ELASTICSEARCH_ENDPOINT}/test_index" +} + +remove_test_index || true +create_test_index +remove_test_index diff --git a/elasticsearch/templates/bin/_verify-repositories.sh.tpl b/elasticsearch/templates/bin/_verify-repositories.sh.tpl new file mode 100644 index 0000000000..d546e52842 --- /dev/null +++ b/elasticsearch/templates/bin/_verify-repositories.sh.tpl @@ -0,0 +1,38 @@ +#!/bin/bash +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{ $envAll := . }} + +set -ex + +function verify_snapshot_repository() { + curl ${CACERT_OPTION} -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ + -XPOST "${ELASTICSEARCH_ENDPOINT}/_snapshot/$1/_verify" +} + +repositories=$(curl ${CACERT_OPTION} -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ + "${ELASTICSEARCH_ENDPOINT}/_snapshot" | jq -r 'keys | @sh') + +repositories=$(echo $repositories | sed "s/'//g") # Strip single quotes from jq output + +for repository in $repositories; do + error=$(verify_snapshot_repository $repository | jq -r '.error' ) + if [ $error == "null" ]; then + echo "$repository is verified." + else + echo "Error for $repository: $(echo $error | jq -r)" + exit 1; + fi +done diff --git a/elasticsearch/templates/certificates.yaml b/elasticsearch/templates/certificates.yaml new file mode 100644 index 0000000000..185f23df21 --- /dev/null +++ b/elasticsearch/templates/certificates.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.certificates -}} +{{ dict "envAll" . "service" "elasticsearch" "type" "internal" | include "helm-toolkit.manifests.certificates" }} +{{- end -}} diff --git a/elasticsearch/templates/configmap-bin-curator.yaml b/elasticsearch/templates/configmap-bin-curator.yaml new file mode 100644 index 0000000000..7f628291f2 --- /dev/null +++ b/elasticsearch/templates/configmap-bin-curator.yaml @@ -0,0 +1,27 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_bin_curator }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: elastic-curator-bin +data: + curator.sh: | +{{ tuple "bin/_curator.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + image-repo-sync.sh: | +{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} +{{- end }} diff --git a/elasticsearch/templates/configmap-bin-elasticsearch.yaml b/elasticsearch/templates/configmap-bin-elasticsearch.yaml new file mode 100644 index 0000000000..645f16d7de --- /dev/null +++ b/elasticsearch/templates/configmap-bin-elasticsearch.yaml @@ -0,0 +1,41 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_bin_elasticsearch }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: elasticsearch-bin +data: + apache.sh: | +{{ tuple "bin/_apache.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + elasticsearch.sh: | +{{ tuple "bin/_elasticsearch.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + helm-tests.sh: | +{{ tuple "bin/_helm-tests.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + ceph-admin-keyring.sh: | +{{ tuple "bin/_ceph-admin-keyring.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + create-s3-bucket.sh: | +{{ tuple "bin/_create_s3_buckets.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + create-s3-user.sh: | +{{ tuple "bin/_create_s3_users.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + create_template.sh: | +{{ tuple "bin/_create_template.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + verify-repositories.sh: | +{{ tuple "bin/_verify-repositories.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + image-repo-sync.sh: | +{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} +{{- end }} diff --git a/elasticsearch/templates/configmap-etc-curator.yaml b/elasticsearch/templates/configmap-etc-curator.yaml new file mode 100644 index 0000000000..b7385a44f7 --- /dev/null +++ b/elasticsearch/templates/configmap-etc-curator.yaml @@ -0,0 +1,27 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_etc_curator }} +{{- $envAll := . }} + +--- +apiVersion: v1 +kind: Secret +metadata: + name: elastic-curator-etc +type: Opaque +data: + action_file.yml: {{ toYaml .Values.conf.curator.action_file | b64enc }} + config.yml: {{ toYaml .Values.conf.curator.config | b64enc }} +{{- end }} diff --git a/elasticsearch/templates/configmap-etc-elasticsearch.yaml b/elasticsearch/templates/configmap-etc-elasticsearch.yaml new file mode 100644 index 0000000000..a81024fe31 --- /dev/null +++ b/elasticsearch/templates/configmap-etc-elasticsearch.yaml @@ -0,0 +1,46 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_etc_elasticsearch }} +{{- $envAll := . }} + +{{- if .Values.conf.elasticsearch.snapshots.enabled }} +{{- range $client, $config := $envAll.Values.storage.s3.clients }} +{{- $settings := $config.settings }} +{{- $endpoint := $settings.endpoint | default (tuple "ceph_object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup") }} +{{- $_ := set $settings "endpoint" $endpoint }} +{{- $protocol := $settings.protocol | default (tuple "ceph_object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup") }} +{{- $_ := set $settings "protocol" $protocol }} +{{- $_:= set $envAll.Values.conf.elasticsearch.config.s3.client $client $settings }} +{{- end -}} +{{- end -}} + +{{- if empty .Values.conf.elasticsearch.config.discovery.seed_hosts -}} +{{- $discovery_svc := tuple "elasticsearch" "discovery" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" -}} +{{- $_:= set .Values.conf.elasticsearch.config.discovery "seed_hosts" $discovery_svc -}} +{{- end -}} + +--- +apiVersion: v1 +kind: Secret +metadata: + name: elasticsearch-etc +type: Opaque +data: + elasticsearch.yml: {{ toYaml .Values.conf.elasticsearch.config | b64enc }} + # NOTE(portdirect): this must be last, to work round helm ~2.7 bug. +{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.httpd "key" "httpd.conf" "format" "Secret") | indent 2 }} +{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.log4j2 "key" "log4j2.properties" "format" "Secret") | indent 2 }} +{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.jvm_options "key" "jvm.options" "format" "Secret") | indent 2 }} +{{- end }} diff --git a/elasticsearch/templates/cron-job-curator.yaml b/elasticsearch/templates/cron-job-curator.yaml new file mode 100644 index 0000000000..475a7442e0 --- /dev/null +++ b/elasticsearch/templates/cron-job-curator.yaml @@ -0,0 +1,108 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.cron_curator }} +{{- $envAll := . }} + +{{- $esUserSecret := .Values.secrets.elasticsearch.user }} + +{{- $serviceAccountName := "elastic-curator" }} +{{ tuple $envAll "curator" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: elastic-curator + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "elasticsearch" "curator" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + schedule: {{ .Values.jobs.curator.cron | quote }} + successfulJobsHistoryLimit: {{ .Values.jobs.curator.history.success }} + failedJobsHistoryLimit: {{ .Values.jobs.curator.history.failed }} + concurrencyPolicy: Forbid + jobTemplate: + metadata: + labels: +{{ tuple $envAll "elasticsearch" "curator" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + spec: + template: + metadata: + labels: +{{ tuple $envAll "elasticsearch" "curator" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 12 }} + spec: +{{ dict "envAll" $envAll "application" "curator" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 10 }} + nodeSelector: + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value | quote }} + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + initContainers: +{{ tuple $envAll "curator" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 12 }} + containers: + - name: curator +{{ tuple $envAll "curator" | include "helm-toolkit.snippets.image" | indent 14 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.curator | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }} +{{ dict "envAll" $envAll "application" "curator" "container" "curator" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 14 }} + command: + - /tmp/curator.sh + env: + - name: ELASTICSEARCH_USERNAME + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_USERNAME + - name: ELASTICSEARCH_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_PASSWORD + - name: ELASTICSEARCH_URL + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_URL + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: pod-etc-curator + mountPath: /etc/config + - name: elastic-curator-bin + mountPath: /tmp/curator.sh + subPath: curator.sh + readOnly: true + - name: elastic-curator-etc + mountPath: /etc/config/config.yml + subPath: config.yml + readOnly: true + - name: elastic-curator-etc + mountPath: /etc/config/action_file.yml + subPath: action_file.yml + readOnly: true +{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.elasticsearch.elasticsearch.internal "path" "/etc/elasticsearch/certs" "certs" tuple "ca.crt" | include "helm-toolkit.snippets.tls_volume_mount" | indent 16 }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: pod-etc-curator + emptyDir: {} + - name: elastic-curator-bin + configMap: + name: elastic-curator-bin + defaultMode: 0555 + - name: elastic-curator-etc + secret: + secretName: elastic-curator-etc + defaultMode: 0444 +{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.elasticsearch.elasticsearch.internal | include "helm-toolkit.snippets.tls_volume" | indent 12 }} +{{- end }} diff --git a/elasticsearch/templates/cron-job-verify-repositories.yaml b/elasticsearch/templates/cron-job-verify-repositories.yaml new file mode 100644 index 0000000000..89c2a2c759 --- /dev/null +++ b/elasticsearch/templates/cron-job-verify-repositories.yaml @@ -0,0 +1,95 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and (.Values.manifests.cron_verify_repositories) (.Values.conf.elasticsearch.snapshots.enabled) }} +{{- $envAll := . }} + +{{- $esUserSecret := .Values.secrets.elasticsearch.user }} + +{{- $serviceAccountName := "verify-repositories" }} +{{ tuple $envAll "verify_repositories" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: elasticsearch-verify-repositories + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "elasticsearch" "verify-repositories" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + schedule: {{ .Values.jobs.verify_repositories.cron | quote }} + successfulJobsHistoryLimit: {{ .Values.jobs.verify_repositories.history.success }} + failedJobsHistoryLimit: {{ .Values.jobs.verify_repositories.history.failed }} + concurrencyPolicy: Forbid + jobTemplate: + metadata: + labels: +{{ tuple $envAll "elasticsearch" "verify-repositories" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ dict "envAll" $envAll "podName" "elasticsearch-verify-repositories" "containerNames" (list "elasticsearch-verify-repositories" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: + template: + metadata: + labels: +{{ tuple $envAll "elasticsearch" "verify-repositories" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 12 }} + spec: +{{ dict "envAll" $envAll "application" "verify_repositories" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 10 }} + nodeSelector: + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value | quote }} + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + initContainers: +{{ tuple $envAll "verify_repositories" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 12 }} + containers: + - name: elasticsearch-verify-repositories +{{ tuple $envAll "snapshot_repository" | include "helm-toolkit.snippets.image" | indent 14 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.snapshot_repository | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }} +{{ dict "envAll" $envAll "application" "verify_repositories" "container" "elasticsearch_verify_repositories" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 14 }} + command: + - /tmp/verify-repositories.sh + env: + - name: ELASTICSEARCH_USERNAME + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_USERNAME + - name: ELASTICSEARCH_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_PASSWORD + - name: ELASTICSEARCH_ENDPOINT + value: {{ printf "%s://%s" (tuple "elasticsearch" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup") (tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup") }} +{{- if .Values.manifests.certificates }} + - name: CACERT_OPTION + value: "--cacert /etc/elasticsearch/certs/ca.crt" +{{- end }} + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: elasticsearch-bin + mountPath: /tmp/verify-repositories.sh + subPath: verify-repositories.sh + readOnly: true +{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.elasticsearch.elasticsearch.internal "path" "/etc/elasticsearch/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 16 }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: elasticsearch-bin + configMap: + name: elasticsearch-bin + defaultMode: 0555 +{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.elasticsearch.elasticsearch.internal | include "helm-toolkit.snippets.tls_volume" | indent 12 }} +{{- end }} diff --git a/elasticsearch/templates/deployment-client.yaml b/elasticsearch/templates/deployment-client.yaml new file mode 100644 index 0000000000..4185975197 --- /dev/null +++ b/elasticsearch/templates/deployment-client.yaml @@ -0,0 +1,234 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "readinessProbeTemplate" }} +{{- $probePort := tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +{{- $probeUser := .Values.endpoints.elasticsearch.auth.admin.username }} +{{- $probePass := .Values.endpoints.elasticsearch.auth.admin.password }} +{{- $authHeader := printf "%s:%s" $probeUser $probePass | b64enc }} +httpGet: + path: /_cluster/health + scheme: {{ tuple "elasticsearch" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" | upper }} + port: {{ $probePort }} + httpHeaders: + - name: Authorization + value: Basic {{ $authHeader }} +{{- end }} +{{- define "livenessProbeTemplate" }} +{{- $probePort := tuple "elasticsearch" "internal" "discovery" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +tcpSocket: + port: {{ $probePort }} +{{- end }} + +{{- if .Values.manifests.deployment_client }} +{{- $envAll := . }} + +{{- $esUserSecret := .Values.secrets.elasticsearch.user }} +{{- $s3UserSecret := .Values.secrets.rgw.elasticsearch }} + +{{- $mounts_elasticsearch := .Values.pod.mounts.elasticsearch.elasticsearch }} + +{{- $serviceAccountName := printf "%s-%s" .Release.Name "elasticsearch-client" }} +{{ tuple $envAll "elasticsearch_client" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: elasticsearch-client + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "elasticsearch" "client" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + replicas: {{ .Values.pod.replicas.client }} + selector: + matchLabels: +{{ tuple $envAll "elasticsearch" "client" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} +{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "elasticsearch" "client" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin-elasticsearch.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-etc-hash: {{ tuple "configmap-etc-elasticsearch.yaml" . | include "helm-toolkit.utils.hash" }} +{{ dict "envAll" $envAll "podName" "elasticsearch-client" "containerNames" (list "elasticsearch-client" "init" "memory-map-increase" "apache-proxy") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "client" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + affinity: +{{ tuple $envAll "elasticsearch" "client" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} + nodeSelector: + {{ .Values.labels.client.node_selector_key }}: {{ .Values.labels.client.node_selector_value | quote }} + terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.client.timeout | default "600" }} + initContainers: +{{ tuple $envAll "elasticsearch_client" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + - name: memory-map-increase +{{ tuple $envAll "memory_init" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.client | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "client" "container" "memory_map_increase" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - sysctl + - -w + - vm.max_map_count={{ .Values.conf.init.max_map_count }} + containers: + - name: apache-proxy +{{ tuple $envAll "apache_proxy" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.apache_proxy | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "client" "container" "apache_proxy" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/apache.sh + - start + ports: + - name: {{ tuple "elasticsearch" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" }} + containerPort: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + readinessProbe: + tcpSocket: + port: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + initialDelaySeconds: 20 + periodSeconds: 10 + env: + - name: ELASTICSEARCH_USERNAME + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_USERNAME + - name: ELASTICSEARCH_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_PASSWORD + - name: ELASTICSEARCH_LOGGING_USERNAME + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_LOGGING_USERNAME + - name: ELASTICSEARCH_LOGGING_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_LOGGING_PASSWORD + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: elasticsearch-bin + mountPath: /tmp/apache.sh + subPath: apache.sh + readOnly: true + - name: elasticsearch-etc + mountPath: /usr/local/apache2/conf/httpd.conf + subPath: httpd.conf + readOnly: true +{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.elasticsearch.elasticsearch.internal "path" "/etc/elasticsearch/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }} + - name: elasticsearch-client +{{ tuple $envAll "elasticsearch" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.client | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "client" "container" "elasticsearch_client" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/elasticsearch.sh + - start + lifecycle: + preStop: + exec: + command: + - /tmp/elasticsearch.sh + - stop + ports: + - name: transport + containerPort: {{ tuple "elasticsearch" "internal" "discovery" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +{{ dict "envAll" . "component" "elasticsearch" "container" "elasticsearch-client" "type" "liveness" "probeTemplate" (include "livenessProbeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }} +{{ dict "envAll" . "component" "elasticsearch" "container" "elasticsearch-client" "type" "readiness" "probeTemplate" (include "readinessProbeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }} + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: node.roles + value: "[ingest]" + - name: HTTP_ENABLE + value: "true" + - name: DISCOVERY_SERVICE + value: {{ tuple "elasticsearch" "discovery" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + - name: ES_JAVA_OPTS + value: "{{ .Values.conf.elasticsearch.env.java_opts.client }}" +{{- if .Values.manifests.certificates }} + - name: JAVA_KEYSTORE_CERT_PATH + value: "/usr/share/elasticsearch/config/ca.crt" + - name: ELASTICSEARCH_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_PASSWORD +{{- end }} +{{- if .Values.conf.elasticsearch.snapshots.enabled }} +{{- if .Values.manifests.object_bucket_claim }} +{{- include "helm-toolkit.snippets.rgw_s3_bucket_user_env_vars_rook" . | indent 12 }} +{{- else }} +{{- include "helm-toolkit.snippets.rgw_s3_user_env_vars" . | indent 12 }} +{{- end }} +{{- end }} +{{- if .Values.pod.env.client }} +{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.client | indent 12 }} +{{- end }} +{{- if .Values.pod.env.secrets }} +{{ tuple $envAll .Values.pod.env.secrets | include "helm-toolkit.utils.to_k8s_env_secret_vars" | indent 12 }} +{{- end }} + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: elasticsearch-logs + mountPath: {{ .Values.conf.elasticsearch.config.path.logs }} + - name: elasticsearch-bin + mountPath: /tmp/elasticsearch.sh + subPath: elasticsearch.sh + readOnly: true + - name: elasticsearch-etc + mountPath: /usr/share/elasticsearch/config/elasticsearch.yml + subPath: elasticsearch.yml + readOnly: true + - name: elasticsearch-etc + mountPath: /usr/share/elasticsearch/config/log4j2.properties + subPath: log4j2.properties + readOnly: true + - name: elasticsearch-etc + mountPath: /usr/share/elasticsearch/config/jvm.options + subPath: jvm.options + readOnly: true + - name: storage + mountPath: {{ .Values.conf.elasticsearch.config.path.data }} +{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.elasticsearch.elasticsearch.internal "path" "/usr/share/elasticsearch/config" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }} +{{ if $mounts_elasticsearch.volumeMounts }}{{ toYaml $mounts_elasticsearch.volumeMounts | indent 12 }}{{ end }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: elasticsearch-logs + emptyDir: {} + - name: elasticsearch-bin + configMap: + name: elasticsearch-bin + defaultMode: 0555 + - name: elasticsearch-etc + secret: + secretName: elasticsearch-etc + defaultMode: 0444 + - name: storage + emptyDir: {} +{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.elasticsearch.elasticsearch.internal | include "helm-toolkit.snippets.tls_volume" | indent 8 }} +{{ if $mounts_elasticsearch.volumes }}{{ toYaml $mounts_elasticsearch.volumes | indent 8 }}{{ end }} +{{- end }} diff --git a/elasticsearch/templates/deployment-gateway.yaml b/elasticsearch/templates/deployment-gateway.yaml new file mode 100644 index 0000000000..bd80aeeba2 --- /dev/null +++ b/elasticsearch/templates/deployment-gateway.yaml @@ -0,0 +1,173 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.network.remote_clustering.enabled }} +{{- $envAll := . }} + +{{- $esUserSecret := .Values.secrets.elasticsearch.user }} +{{- $s3UserSecret := .Values.secrets.rgw.elasticsearch }} + +{{- $mounts_elasticsearch := .Values.pod.mounts.elasticsearch.elasticsearch }} + +{{- $serviceAccountName := printf "%s-%s" .Release.Name "elasticsearch-remote-gateway" }} +{{ tuple $envAll "elasticsearch_gateway" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: elasticsearch-gateway + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "elasticsearch" "gateway" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: +{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }} + replicas: {{ .Values.pod.replicas.gateway }} + selector: + matchLabels: +{{ tuple $envAll "elasticsearch" "gateway" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} +{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "elasticsearch" "gateway" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin-elasticsearch.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-etc-hash: {{ tuple "configmap-etc-elasticsearch.yaml" . | include "helm-toolkit.utils.hash" }} +{{ dict "envAll" $envAll "podName" "elasticsearch-gateway" "containerNames" (list "elasticsearch-remote-gateway") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "gateway" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + affinity: +{{ tuple $envAll "elasticsearch" "gateway" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} + nodeSelector: + {{ .Values.labels.gateway.node_selector_key }}: {{ .Values.labels.gateway.node_selector_value | quote }} + terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.client.timeout | default "600" }} + initContainers: +{{ tuple $envAll "elasticsearch" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + - name: memory-map-increase +{{ tuple $envAll "memory_init" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.client | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "gateway" "container" "memory_map_increase" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - sysctl + - -w + - vm.max_map_count={{ .Values.conf.init.max_map_count }} + containers: + - name: elasticsearch-gateway +{{ tuple $envAll "elasticsearch" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.gateway | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "gateway" "container" "elasticsearch_gateway" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/elasticsearch.sh + - start + lifecycle: + preStop: + exec: + command: + - /tmp/elasticsearch.sh + - stop + ports: + - name: transport + containerPort: {{ tuple "elasticsearch" "internal" "discovery" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + livenessProbe: + tcpSocket: + port: {{ tuple "elasticsearch" "internal" "discovery" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + initialDelaySeconds: 20 + periodSeconds: 10 + readinessProbe: + tcpSocket: + port: {{ tuple "elasticsearch" "internal" "discovery" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + initialDelaySeconds: 20 + periodSeconds: 10 + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: node.roles + value: "[ingest]" + - name: HTTP_ENABLE + value: "false" + - name: DISCOVERY_SERVICE + value: {{ tuple "elasticsearch" "discovery" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + - name: ES_JAVA_OPTS + value: "{{ .Values.conf.elasticsearch.env.java_opts.client }}" +{{- if .Values.manifests.certificates }} + - name: JAVA_KEYSTORE_CERT_PATH + value: "/usr/share/elasticsearch/config/ca.crt" + - name: ELASTICSEARCH_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_PASSWORD +{{- end }} +{{- if .Values.conf.elasticsearch.snapshots.enabled }} +{{- if .Values.manifests.object_bucket_claim }} +{{- include "helm-toolkit.snippets.rgw_s3_bucket_user_env_vars_rook" . | indent 12 }} +{{- else }} +{{- include "helm-toolkit.snippets.rgw_s3_user_env_vars" . | indent 12 }} +{{- end }} +{{- end }} +{{- if .Values.pod.env.gateway }} +{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.gateway | indent 12 }} +{{- end }} + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: elasticsearch-logs + mountPath: {{ .Values.conf.elasticsearch.config.path.logs }} + - name: elasticsearch-bin + mountPath: /tmp/elasticsearch.sh + subPath: elasticsearch.sh + readOnly: true + - name: elasticsearch-etc + mountPath: /usr/share/elasticsearch/config/elasticsearch.yml + subPath: elasticsearch.yml + readOnly: true + - name: elasticsearch-etc + mountPath: /usr/share/elasticsearch/config/log4j2.properties + subPath: log4j2.properties + readOnly: true + - name: elasticsearch-etc + mountPath: /usr/share/elasticsearch/config/jvm.options + subPath: jvm.options + readOnly: true + - name: storage + mountPath: {{ .Values.conf.elasticsearch.config.path.data }} +{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.elasticsearch.elasticsearch.internal "path" "/usr/share/elasticsearch/config" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }} +{{ if $mounts_elasticsearch.volumeMounts }}{{ toYaml $mounts_elasticsearch.volumeMounts | indent 12 }}{{ end }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: elasticsearch-logs + emptyDir: {} + - name: elasticsearch-bin + configMap: + name: elasticsearch-bin + defaultMode: 0555 + - name: elasticsearch-etc + secret: + secretName: elasticsearch-etc + defaultMode: 0444 + - name: storage + emptyDir: {} +{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.elasticsearch.elasticsearch.internal | include "helm-toolkit.snippets.tls_volume" | indent 8 }} +{{ if $mounts_elasticsearch.volumes }}{{ toYaml $mounts_elasticsearch.volumes | indent 8 }}{{ end }} +{{- end }} diff --git a/elasticsearch/templates/ingress-elasticsearch.yaml b/elasticsearch/templates/ingress-elasticsearch.yaml new file mode 100644 index 0000000000..4e73b02c20 --- /dev/null +++ b/elasticsearch/templates/ingress-elasticsearch.yaml @@ -0,0 +1,24 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.ingress .Values.network.elasticsearch.ingress.public }} +{{- $envAll := . -}} +{{- $port := tuple "elasticsearch" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" }} +{{- $ingressOpts := dict "envAll" $envAll "backendService" "elasticsearch" "backendServiceType" "elasticsearch" "backendPort" $port -}} +{{- $secretName := $envAll.Values.secrets.tls.elasticsearch.elasticsearch.internal -}} +{{- if and .Values.manifests.certificates $secretName -}} +{{- $_ := set $ingressOpts "certIssuer" .Values.endpoints.elasticsearch.host_fqdn_override.default.tls.issuerRef.name -}} +{{- end -}} +{{ $ingressOpts | include "helm-toolkit.manifests.ingress" }} +{{- end }} diff --git a/elasticsearch/templates/job-elasticsearch-template.yaml b/elasticsearch/templates/job-elasticsearch-template.yaml new file mode 100644 index 0000000000..768c60650b --- /dev/null +++ b/elasticsearch/templates/job-elasticsearch-template.yaml @@ -0,0 +1,91 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.job_elasticsearch_templates }} +{{- $envAll := . }} +{{- $esUserSecret := .Values.secrets.elasticsearch.user }} +{{- $mounts_elasticsearch_templates := .Values.pod.mounts.elasticsearch_templates.elasticsearch_templates }} +{{- $mounts_elasticsearch_templates_init := .Values.pod.mounts.elasticsearch_templates.init_container }} + +{{- $serviceAccountName := "create-elasticsearch-templates" }} +{{ tuple $envAll "elasticsearch_templates" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: create-elasticsearch-templates + labels: +{{ tuple $envAll "elasticsearch" "create-templates" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +spec: + backoffLimit: {{ .Values.jobs.create_elasticsearch_templates.backoffLimit }} + template: + metadata: + labels: +{{ tuple $envAll "elasticsearch" "create-templates" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} +{{ dict "envAll" $envAll "podName" "create-elasticsearch-templates" "containerNames" (list "create-elasticsearch-templates" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "create_template" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value | quote }} + initContainers: +{{ tuple $envAll "elasticsearch_templates" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: create-elasticsearch-templates +{{ tuple $envAll "elasticsearch_templates" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.elasticsearch_templates | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "create_template" "container" "create_elasticsearch_template" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + env: + - name: ELASTICSEARCH_ENDPOINT + value: {{ printf "%s://%s" (tuple "elasticsearch" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup") (tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup") }} +{{- if .Values.manifests.certificates }} + - name: CACERT_OPTION + value: "--cacert /etc/elasticsearch/certs/ca.crt" +{{- end }} + - name: ELASTICSEARCH_USERNAME + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_USERNAME + - name: ELASTICSEARCH_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_PASSWORD + command: + - /tmp/create_template.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: elasticsearch-bin + mountPath: /tmp/create_template.sh + subPath: create_template.sh + readOnly: true +{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.elasticsearch.elasticsearch.internal "path" "/etc/elasticsearch/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }} +{{ if $mounts_elasticsearch_templates.volumeMounts }}{{ toYaml $mounts_elasticsearch_templates.volumeMounts | indent 12 }}{{ end }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: elasticsearch-bin + configMap: + name: elasticsearch-bin + defaultMode: 0555 +{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.elasticsearch.elasticsearch.internal | include "helm-toolkit.snippets.tls_volume" | indent 8 }} +{{ if $mounts_elasticsearch_templates.volumes }}{{ toYaml $mounts_elasticsearch_templates.volumes | indent 8 }}{{ end }} +{{- end }} diff --git a/elasticsearch/templates/job-image-repo-sync.yaml b/elasticsearch/templates/job-image-repo-sync.yaml new file mode 100644 index 0000000000..ec74fad4ee --- /dev/null +++ b/elasticsearch/templates/job-image-repo-sync.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "elasticsearch" -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/elasticsearch/templates/job-s3-bucket.yaml b/elasticsearch/templates/job-s3-bucket.yaml new file mode 100644 index 0000000000..8ea633d8d4 --- /dev/null +++ b/elasticsearch/templates/job-s3-bucket.yaml @@ -0,0 +1,23 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and (.Values.manifests.job_s3_bucket) (.Values.conf.elasticsearch.snapshots.enabled) }} +{{- $esBucket := .Values.conf.elasticsearch.snapshots.bucket }} +{{- $s3BucketJob := dict "envAll" . "serviceName" "elasticsearch" "s3Bucket" $esBucket -}} +{{- if .Values.manifests.certificates }} +{{- $_ := set $s3BucketJob "tlsCertificateSecret" .Values.secrets.tls.elasticsearch.elasticsearch.internal -}} +{{- $_ := set $s3BucketJob "tlsCertificatePath" "/etc/elasticsearch/certs/ca.crt" -}} +{{- end }} +{{ $s3BucketJob | include "helm-toolkit.manifests.job_s3_bucket" }} +{{- end -}} diff --git a/elasticsearch/templates/job-s3-user.yaml b/elasticsearch/templates/job-s3-user.yaml new file mode 100644 index 0000000000..8fcb32e076 --- /dev/null +++ b/elasticsearch/templates/job-s3-user.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and (.Values.manifests.job_s3_user) (.Values.conf.elasticsearch.snapshots.enabled) }} +{{- $s3UserJob := dict "envAll" . "serviceName" "elasticsearch" -}} +{{ $s3UserJob | include "helm-toolkit.manifests.job_s3_user" }} +{{- end }} diff --git a/elasticsearch/templates/monitoring/prometheus/exporter-deployment.yaml b/elasticsearch/templates/monitoring/prometheus/exporter-deployment.yaml new file mode 100644 index 0000000000..61d6f978c1 --- /dev/null +++ b/elasticsearch/templates/monitoring/prometheus/exporter-deployment.yaml @@ -0,0 +1,120 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.monitoring.prometheus.deployment_exporter .Values.monitoring.prometheus.enabled }} +{{- $envAll := . }} + +{{- $esUserSecret := .Values.secrets.elasticsearch.user }} + +{{- $serviceAccountName := "prometheus-elasticsearch-exporter" }} +{{ tuple $envAll "prometheus_elasticsearch_exporter" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus-elasticsearch-exporter + labels: +{{ tuple $envAll "prometheus-elasticsearch-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + replicas: {{ .Values.pod.replicas.prometheus_elasticsearch_exporter }} + selector: + matchLabels: +{{ tuple $envAll "prometheus-elasticsearch-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} +{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "prometheus-elasticsearch-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} +{{ dict "envAll" $envAll "podName" "prometheus-elasticsearch-exporter" "containerNames" (list "elasticsearch-exporter" "init" ) | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "exporter" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + nodeSelector: + {{ .Values.labels.exporter.node_selector_key }}: {{ .Values.labels.exporter.node_selector_value | quote }} + terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.prometheus_elasticsearch_exporter.timeout | default "30" }} + initContainers: +{{ tuple $envAll "prometheus_elasticsearch_exporter" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: elasticsearch-exporter +{{ tuple $envAll "prometheus_elasticsearch_exporter" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.exporter | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "exporter" "container" "elasticsearch_exporter" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - "elasticsearch_exporter" + - '--es.uri=$(ELASTICSEARCH_URI)' + - '--web.telemetry-path={{ .Values.endpoints.prometheus_elasticsearch_exporter.path.default }}' + - '--web.listen-address=:{{ .Values.endpoints.prometheus_elasticsearch_exporter.port.metrics.default }}' + - '--es.timeout={{ .Values.conf.prometheus_elasticsearch_exporter.es.timeout }}' + - '--log.format={{ .Values.conf.prometheus_elasticsearch_exporter.log.format }}' + - '--log.level={{ .Values.conf.prometheus_elasticsearch_exporter.log.level }}' + {{- if .Values.conf.prometheus_elasticsearch_exporter.es.all }} + - '--es.all' + {{- end }} + {{- if .Values.conf.prometheus_elasticsearch_exporter.es.indices }} + - '--es.indices' + {{- end }} + {{- if .Values.conf.prometheus_elasticsearch_exporter.es.indices_settings }} + - '--es.indices_settings' + {{- end }} + {{- if .Values.conf.prometheus_elasticsearch_exporter.es.indices_mappings }} + - '--es.indices_mappings' + {{- end }} + {{- if .Values.conf.prometheus_elasticsearch_exporter.es.aliases }} + - '--es.aliases' + {{- end }} + {{- if .Values.conf.prometheus_elasticsearch_exporter.es.shards }} + - '--es.shards' + {{- end }} + {{- if .Values.conf.prometheus_elasticsearch_exporter.es.snapshots }} + - '--collector.snapshots' + {{- end }} + {{- if .Values.conf.prometheus_elasticsearch_exporter.es.cluster_settings }} + - '--collector.clustersettings' + {{- end }} + {{- if .Values.conf.prometheus_elasticsearch_exporter.es.slm }} + - '--es.slm' + {{- end }} + {{- if .Values.conf.prometheus_elasticsearch_exporter.es.data_stream }} + - '--es.data_stream' + {{- end }} + {{- if .Values.manifests.certificates }} + - '--es.ca=/tmp/elasticsearch/certs/ca.crt' + {{- else }} + - '--es.ssl-skip-verify' + {{- end }} + env: + - name: ELASTICSEARCH_URI + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_URI + ports: + - name: metrics + containerPort: {{ tuple "prometheus_elasticsearch_exporter" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + readinessProbe: + tcpSocket: + port: {{ tuple "prometheus_elasticsearch_exporter" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + initialDelaySeconds: 20 + periodSeconds: 10 + volumeMounts: + - name: pod-tmp + mountPath: /tmp +{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.elasticsearch.elasticsearch.internal "path" "/tmp/elasticsearch/certs" "certs" tuple "ca.crt" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }} + volumes: + - name: pod-tmp + emptyDir: {} +{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.elasticsearch.elasticsearch.internal | include "helm-toolkit.snippets.tls_volume" | indent 8 }} +{{- end }} diff --git a/elasticsearch/templates/monitoring/prometheus/exporter-network-policy.yaml b/elasticsearch/templates/monitoring/prometheus/exporter-network-policy.yaml new file mode 100644 index 0000000000..131a261ff4 --- /dev/null +++ b/elasticsearch/templates/monitoring/prometheus/exporter-network-policy.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.monitoring.prometheus.network_policy_exporter .Values.monitoring.prometheus.enabled -}} +{{- $netpol_opts := dict "envAll" . "name" "application" "label" "prometheus-elasticsearch-exporter" -}} +{{ $netpol_opts | include "helm-toolkit.manifests.kubernetes_network_policy" }} +{{- end -}} diff --git a/elasticsearch/templates/monitoring/prometheus/exporter-service.yaml b/elasticsearch/templates/monitoring/prometheus/exporter-service.yaml new file mode 100644 index 0000000000..ecad51c016 --- /dev/null +++ b/elasticsearch/templates/monitoring/prometheus/exporter-service.yaml @@ -0,0 +1,35 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.monitoring.prometheus.service_exporter .Values.monitoring.prometheus.enabled }} +{{- $envAll := . }} +{{- $prometheus_annotations := $envAll.Values.monitoring.prometheus.elasticsearch_exporter }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "prometheus_elasticsearch_exporter" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + labels: +{{ tuple $envAll "prometheus-elasticsearch-exporter" "metrics" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: +{{- if .Values.monitoring.prometheus.enabled }} +{{ tuple $prometheus_annotations | include "helm-toolkit.snippets.prometheus_service_annotations" | indent 4 }} +{{- end }} +spec: + ports: + - name: metrics + port: {{ tuple "prometheus_elasticsearch_exporter" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + selector: +{{ tuple $envAll "prometheus-elasticsearch-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- end }} diff --git a/elasticsearch/templates/network-policy.yaml b/elasticsearch/templates/network-policy.yaml new file mode 100644 index 0000000000..f0b18b5150 --- /dev/null +++ b/elasticsearch/templates/network-policy.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.network_policy -}} +{{- $netpol_opts := dict "envAll" . "name" "application" "label" "elasticsearch" -}} +{{ $netpol_opts | include "helm-toolkit.manifests.kubernetes_network_policy" }} +{{- end -}} diff --git a/elasticsearch/templates/object-bucket-claim.yaml b/elasticsearch/templates/object-bucket-claim.yaml new file mode 100644 index 0000000000..f53a0a2b32 --- /dev/null +++ b/elasticsearch/templates/object-bucket-claim.yaml @@ -0,0 +1,61 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and (.Values.manifests.object_bucket_claim) (.Values.conf.elasticsearch.snapshots.enabled) }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: elasticsearch-dependencies-objectbucket + namespace: {{ .Release.Namespace }} +rules: + - apiGroups: + - "objectbucket.io" + verbs: + - get + - list + resources: + - objectbuckets +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: elasticsearch-dependencies-objectbucket + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: elasticsearch-dependencies-objectbucket +subjects: + - kind: ServiceAccount + name: create-elasticsearch-templates + namespace: {{ .Release.Namespace }} + - kind: ServiceAccount + name: verify-repositories + namespace: {{ .Release.Namespace }} + +{{- range $bucket := .Values.storage.s3.buckets }} +# When using this Rook CRD, not only bucket will be created, +# but also a secret containing the credentials to access the bucket. +--- +apiVersion: objectbucket.io/v1alpha1 +kind: ObjectBucketClaim +metadata: + name: {{ $bucket.name }} +spec: + bucketName: {{ $bucket.name }} + storageClassName: {{ $bucket.storage_class }} +... +{{- end -}} +{{- end -}} diff --git a/elasticsearch/templates/pod-helm-tests.yaml b/elasticsearch/templates/pod-helm-tests.yaml new file mode 100644 index 0000000000..75e2de2428 --- /dev/null +++ b/elasticsearch/templates/pod-helm-tests.yaml @@ -0,0 +1,80 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.helm_tests }} +{{- $envAll := . }} +{{- $esUserSecret := .Values.secrets.elasticsearch.user }} + +{{- $serviceAccountName := print .Release.Name "-test" }} +{{ tuple $envAll "tests" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: v1 +kind: Pod +metadata: + name: "{{.Release.Name}}-test" + labels: +{{ tuple $envAll "elasticsearch" "test" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: + "helm.sh/hook": test-success + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +{{ dict "envAll" $envAll "podName" "elasticsearch-test" "containerNames" (list "init" "elasticsearch-helm-tests") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 4 }} +spec: +{{ dict "envAll" $envAll "application" "test" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 2 }} + serviceAccountName: {{ $serviceAccountName }} + nodeSelector: + {{ .Values.labels.test.node_selector_key }}: {{ .Values.labels.test.node_selector_value }} + restartPolicy: Never + initContainers: +{{ tuple $envAll "tests" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 4 }} + containers: + - name: elasticsearch-helm-tests +{{ tuple $envAll "helm_tests" | include "helm-toolkit.snippets.image" | indent 6 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.tests | include "helm-toolkit.snippets.kubernetes_resources" | indent 6 }} +{{ dict "envAll" $envAll "application" "test" "container" "helm_tests" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 6 }} + command: + - /tmp/helm-tests.sh + env: + - name: ELASTICSEARCH_USERNAME + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_USERNAME + - name: ELASTICSEARCH_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_PASSWORD + - name: ELASTICSEARCH_ENDPOINT + value: {{ printf "%s://%s" (tuple "elasticsearch" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup") (tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup") }} +{{- if .Values.manifests.certificates }} + - name: CACERT_OPTION + value: "--cacert /etc/elasticsearch/certs/ca.crt" +{{- end }} + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: elasticsearch-bin + mountPath: /tmp/helm-tests.sh + subPath: helm-tests.sh + readOnly: true +{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.elasticsearch.elasticsearch.internal "path" "/etc/elasticsearch/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 8 }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: elasticsearch-bin + configMap: + name: elasticsearch-bin + defaultMode: 0555 +{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.elasticsearch.elasticsearch.internal | include "helm-toolkit.snippets.tls_volume" | indent 4 }} +{{- end }} diff --git a/elasticsearch/templates/secret-elasticsearch.yaml b/elasticsearch/templates/secret-elasticsearch.yaml new file mode 100644 index 0000000000..acbd20e146 --- /dev/null +++ b/elasticsearch/templates/secret-elasticsearch.yaml @@ -0,0 +1,40 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_elasticsearch }} +{{- $envAll := . }} +{{- $secretName := index $envAll.Values.secrets.elasticsearch.user }} + +{{- $elasticsearch_user := .Values.endpoints.elasticsearch.auth.admin.username }} +{{- $elasticsearch_password := .Values.endpoints.elasticsearch.auth.admin.password }} +{{- $elasticsearch_host := tuple "elasticsearch" "internal" "http" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} +{{- $elasticsearch_scheme := tuple "elasticsearch" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" }} +{{- $elasticsearch_uri := printf "%s://%s:%s@%s" $elasticsearch_scheme $elasticsearch_user $elasticsearch_password $elasticsearch_host }} +{{- $elasticsearch_url := printf "%s://%s" $elasticsearch_scheme $elasticsearch_host }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} +type: Opaque +data: + ELASTICSEARCH_USERNAME: {{ .Values.endpoints.elasticsearch.auth.admin.username | b64enc }} + ELASTICSEARCH_PASSWORD: {{ .Values.endpoints.elasticsearch.auth.admin.password | b64enc }} + ELASTICSEARCH_LOGGING_USERNAME: {{ .Values.endpoints.elasticsearch.auth.logging.username | b64enc }} + ELASTICSEARCH_LOGGING_PASSWORD: {{ .Values.endpoints.elasticsearch.auth.logging.password | b64enc }} + ELASTICSEARCH_URI: {{ $elasticsearch_uri | b64enc }} + ELASTICSEARCH_URL: {{ $elasticsearch_url | b64enc }} + BIND_DN: {{ .Values.endpoints.ldap.auth.admin.bind | b64enc }} + BIND_PASSWORD: {{ .Values.endpoints.ldap.auth.admin.password | b64enc }} +{{- end }} diff --git a/elasticsearch/templates/secret-environment.yaml b/elasticsearch/templates/secret-environment.yaml new file mode 100644 index 0000000000..58fc1b41ee --- /dev/null +++ b/elasticsearch/templates/secret-environment.yaml @@ -0,0 +1,27 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_environment .Values.pod.env.secrets }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ printf "%s-%s" $envAll.Release.Name "env-secret" | quote }} +type: Opaque +data: + {{- range $key, $value := .Values.pod.env.secrets }} + {{ $key | upper }}: {{ $value | b64enc }} + {{- end }} +{{- end }} diff --git a/elasticsearch/templates/secret-ingress-tls.yaml b/elasticsearch/templates/secret-ingress-tls.yaml new file mode 100644 index 0000000000..d739cdc257 --- /dev/null +++ b/elasticsearch/templates/secret-ingress-tls.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_ingress_tls }} +{{- include "helm-toolkit.manifests.secret_ingress_tls" ( dict "envAll" . "backendServiceType" "elasticsearch" "backendService" "elasticsearch" ) }} +{{- end }} diff --git a/elasticsearch/templates/secret-registry.yaml b/elasticsearch/templates/secret-registry.yaml new file mode 100644 index 0000000000..da979b3223 --- /dev/null +++ b/elasticsearch/templates/secret-registry.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_registry .Values.endpoints.oci_image_registry.auth.enabled }} +{{ include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) }} +{{- end }} diff --git a/elasticsearch/templates/secret-s3-user.yaml b/elasticsearch/templates/secret-s3-user.yaml new file mode 100644 index 0000000000..51ed46809a --- /dev/null +++ b/elasticsearch/templates/secret-s3-user.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_s3 }} +{{ include "helm-toolkit.snippets.rgw_s3_secret_creds" . }} +{{- end }} diff --git a/elasticsearch/templates/service-data.yaml b/elasticsearch/templates/service-data.yaml new file mode 100644 index 0000000000..806e1a4185 --- /dev/null +++ b/elasticsearch/templates/service-data.yaml @@ -0,0 +1,28 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.service_data }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "elasticsearch" "data" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +spec: + ports: + - name: transport + port: {{ tuple "elasticsearch" "internal" "discovery" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + selector: +{{ tuple $envAll "elasticsearch" "data" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- end }} diff --git a/elasticsearch/templates/service-discovery.yaml b/elasticsearch/templates/service-discovery.yaml new file mode 100644 index 0000000000..6c9f01765e --- /dev/null +++ b/elasticsearch/templates/service-discovery.yaml @@ -0,0 +1,28 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.service_discovery }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "elasticsearch" "discovery" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +spec: + ports: + - name: transport + port: {{ tuple "elasticsearch" "internal" "discovery" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + selector: +{{ tuple $envAll "elasticsearch" "master" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- end }} diff --git a/elasticsearch/templates/service-gateway.yaml b/elasticsearch/templates/service-gateway.yaml new file mode 100644 index 0000000000..27b4f1de4c --- /dev/null +++ b/elasticsearch/templates/service-gateway.yaml @@ -0,0 +1,30 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.network.remote_clustering.enabled }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "elasticsearch" "gateway" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +spec: + ports: + - name: transport + port: {{ tuple "elasticsearch" "internal" "discovery" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + nodePort: {{ .Values.network.remote_clustering.node_port.port }} + selector: +{{ tuple $envAll "elasticsearch" "gateway" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + type: NodePort +{{- end }} diff --git a/elasticsearch/templates/service-ingress-elasticsearch.yaml b/elasticsearch/templates/service-ingress-elasticsearch.yaml new file mode 100644 index 0000000000..325852ee13 --- /dev/null +++ b/elasticsearch/templates/service-ingress-elasticsearch.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.service_ingress .Values.network.elasticsearch.ingress.public }} +{{- $serviceIngressOpts := dict "envAll" . "backendServiceType" "elasticsearch" -}} +{{ $serviceIngressOpts | include "helm-toolkit.manifests.service_ingress" }} +{{- end }} diff --git a/elasticsearch/templates/service-logging.yaml b/elasticsearch/templates/service-logging.yaml new file mode 100644 index 0000000000..c8dd1d0fbb --- /dev/null +++ b/elasticsearch/templates/service-logging.yaml @@ -0,0 +1,35 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.service_logging }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "elasticsearch" "default" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +spec: + ports: + - name: {{ tuple "elasticsearch" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" }} + port: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + targetPort: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + {{- if .Values.network.elasticsearch.node_port.enabled }} + nodePort: {{ .Values.network.elasticsearch.node_port.port }} + {{- end }} + selector: +{{ tuple $envAll "elasticsearch" "client" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + {{- if .Values.network.elasticsearch.node_port.enabled }} + type: NodePort + {{- end }} +{{- end }} diff --git a/elasticsearch/templates/statefulset-data.yaml b/elasticsearch/templates/statefulset-data.yaml new file mode 100644 index 0000000000..2f95a6080d --- /dev/null +++ b/elasticsearch/templates/statefulset-data.yaml @@ -0,0 +1,199 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.statefulset_data }} +{{- $envAll := . }} + +{{- $esUserSecret := .Values.secrets.elasticsearch.user }} +{{- $s3UserSecret := .Values.secrets.rgw.elasticsearch }} + +{{- $mounts_elasticsearch := .Values.pod.mounts.elasticsearch.elasticsearch }} + +{{- $serviceAccountName := printf "%s-%s" .Release.Name "elasticsearch-data" }} +{{ tuple $envAll "elasticsearch_data" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: elasticsearch-data + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "elasticsearch" "data" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: +{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_statefulset" | indent 2 }} + serviceName: {{ tuple "elasticsearch" "data" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + podManagementPolicy: "Parallel" + replicas: {{ .Values.pod.replicas.data }} + selector: + matchLabels: +{{ tuple $envAll "elasticsearch" "data" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} + template: + metadata: + labels: +{{ tuple $envAll "elasticsearch" "data" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ dict "envAll" $envAll "podName" "elasticsearch-data" "containerNames" (list "elasticsearch-data" "init" "memory-map-increase") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin-elasticsearch.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-etc-hash: {{ tuple "configmap-etc-elasticsearch.yaml" . | include "helm-toolkit.utils.hash" }} +{{- if and .Values.manifests.secret_s3 .Values.conf.elasticsearch.snapshots.enabled }} + secret-s3-user-hash: {{ tuple "secret-s3-user.yaml" . | include "helm-toolkit.utils.hash" }} +{{- end }} + spec: +{{ dict "envAll" $envAll "application" "data" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + affinity: +{{ tuple $envAll "elasticsearch" "data" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} + nodeSelector: + {{ .Values.labels.data.node_selector_key }}: {{ .Values.labels.data.node_selector_value | quote }} + terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.data.timeout | default "600" }} + initContainers: +{{ tuple $envAll "elasticsearch_data" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + - name: memory-map-increase +{{ tuple $envAll "memory_init" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.data | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "data" "container" "memory_map_increase" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - sysctl + - -w + - vm.max_map_count={{ .Values.conf.init.max_map_count }} + - name: elasticsearch-perms +{{ tuple $envAll "elasticsearch" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.prometheus | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "data" "container" "elasticsearch_perms" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - chown + - -R + - "1000:1000" + - {{ .Values.conf.elasticsearch.config.path.data }} + volumeMounts: + - name: storage + mountPath: {{ .Values.conf.elasticsearch.config.path.data }} + containers: + - name: elasticsearch-data +{{ tuple $envAll "elasticsearch" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.data | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "data" "container" "elasticsearch_data" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/elasticsearch.sh + - start_data_node + ports: + - name: transport + containerPort: {{ tuple "elasticsearch" "internal" "discovery" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + readinessProbe: + tcpSocket: + port: {{ tuple "elasticsearch" "internal" "discovery" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + initialDelaySeconds: 20 + periodSeconds: 10 + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: ELASTICSEARCH_USERNAME + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_USERNAME + - name: ELASTICSEARCH_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_PASSWORD + - name: ELASTICSEARCH_ENDPOINT + value: {{ printf "%s://%s" (tuple "elasticsearch" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup") (tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup") }} +{{- if .Values.manifests.certificates }} + - name: CACERT_OPTION + value: "--cacert /usr/share/elasticsearch/config/ca.crt" + - name: JAVA_KEYSTORE_CERT_PATH + value: "/usr/share/elasticsearch/config/ca.crt" +{{- end }} + - name: node.roles + value: "[data]" + - name: HTTP_ENABLE + value: "false" + - name: ES_JAVA_OPTS + value: "{{ .Values.conf.elasticsearch.env.java_opts.data }}" + - name: DISCOVERY_SERVICE + value: {{ tuple "elasticsearch" "discovery" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +{{- if .Values.conf.elasticsearch.snapshots.enabled }} +{{- if .Values.manifests.object_bucket_claim }} +{{- include "helm-toolkit.snippets.rgw_s3_bucket_user_env_vars_rook" . | indent 12 }} +{{- else }} +{{- include "helm-toolkit.snippets.rgw_s3_user_env_vars" . | indent 12 }} +{{- end }} +{{- end }} +{{- if .Values.pod.env.data }} +{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.data | indent 12 }} +{{- end }} + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: elasticsearch-logs + mountPath: {{ .Values.conf.elasticsearch.config.path.logs }} + - name: elasticsearch-bin + mountPath: /tmp/elasticsearch.sh + subPath: elasticsearch.sh + readOnly: true + - name: elasticsearch-etc + mountPath: /usr/share/elasticsearch/config/elasticsearch.yml + subPath: elasticsearch.yml + readOnly: true + - name: elasticsearch-etc + mountPath: /usr/share/elasticsearch/config/log4j2.properties + subPath: log4j2.properties + readOnly: true + - name: elasticsearch-etc + mountPath: /usr/share/elasticsearch/config/jvm.options + subPath: jvm.options + readOnly: true + - name: storage + mountPath: {{ .Values.conf.elasticsearch.config.path.data }} +{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.elasticsearch.elasticsearch.internal "path" "/usr/share/elasticsearch/config" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }} +{{ if $mounts_elasticsearch.volumeMounts }}{{ toYaml $mounts_elasticsearch.volumeMounts | indent 12 }}{{ end }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: elasticsearch-logs + emptyDir: {} + - name: elasticsearch-bin + configMap: + name: elasticsearch-bin + defaultMode: 0555 + - name: elasticsearch-etc + secret: + secretName: elasticsearch-etc + defaultMode: 0444 +{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.elasticsearch.elasticsearch.internal | include "helm-toolkit.snippets.tls_volume" | indent 8 }} +{{ if $mounts_elasticsearch.volumes }}{{ toYaml $mounts_elasticsearch.volumes | indent 8 }}{{ end }} +{{- if not .Values.storage.data.enabled }} + - name: storage + emptyDir: {} +{{- else }} + volumeClaimTemplates: + - metadata: + name: storage + spec: + accessModes: {{ .Values.storage.data.pvc.access_mode }} + resources: + requests: + storage: {{ .Values.storage.data.requests.storage }} + storageClassName: {{ .Values.storage.data.storage_class }} +{{- end }} +{{- end }} diff --git a/elasticsearch/templates/statefulset-master.yaml b/elasticsearch/templates/statefulset-master.yaml new file mode 100644 index 0000000000..c9efbef9ca --- /dev/null +++ b/elasticsearch/templates/statefulset-master.yaml @@ -0,0 +1,193 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.statefulset_master }} +{{- $envAll := . }} + +{{- $mounts_elasticsearch := .Values.pod.mounts.elasticsearch.elasticsearch }} + +{{- $serviceAccountName := "elasticsearch-master" }} +{{ tuple $envAll "elasticsearch_master" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: elasticsearch-master + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "elasticsearch" "master" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + serviceName: {{ tuple "elasticsearch" "discovery" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + podManagementPolicy: "Parallel" + replicas: {{ .Values.pod.replicas.master }} + selector: + matchLabels: +{{ tuple $envAll "elasticsearch" "master" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} +{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_statefulset" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "elasticsearch" "master" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin-elasticsearch.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-etc-hash: {{ tuple "configmap-etc-elasticsearch.yaml" . | include "helm-toolkit.utils.hash" }} +{{- if and .Values.manifests.secret_s3 .Values.conf.elasticsearch.snapshots.enabled }} + secret-s3-user-hash: {{ tuple "secret-s3-user.yaml" . | include "helm-toolkit.utils.hash" }} +{{- end }} +{{ dict "envAll" $envAll "podName" "elasticsearch-master" "containerNames" (list "elasticsearch-master" "init" "memory-map-increase") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "master" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + affinity: +{{ tuple $envAll "elasticsearch" "master" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} + terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.master.timeout | default "600" }} + nodeSelector: + {{ .Values.labels.master.node_selector_key }}: {{ .Values.labels.master.node_selector_value | quote }} + initContainers: +{{ tuple $envAll "elasticsearch_master" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + - name: memory-map-increase +{{ tuple $envAll "memory_init" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.master | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "master" "container" "memory_map_increase" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - sysctl + - -w + - vm.max_map_count={{ .Values.conf.init.max_map_count }} + - name: elasticsearch-perms +{{ tuple $envAll "elasticsearch" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.prometheus | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "master" "container" "elasticsearch_perms" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - chown + - -R + - "1000:1000" + - {{ .Values.conf.elasticsearch.config.path.data }} + volumeMounts: + - name: storage + mountPath: {{ .Values.conf.elasticsearch.config.path.data }} + containers: + - name: elasticsearch-master +{{ dict "envAll" $envAll "application" "master" "container" "elasticsearch_master" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} +{{ tuple $envAll "elasticsearch" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.master | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + command: + - /tmp/elasticsearch.sh + - start_master_node + lifecycle: + preStop: + exec: + command: + - /tmp/elasticsearch.sh + - stop + ports: + - name: transport + containerPort: {{ tuple "elasticsearch" "internal" "discovery" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + readinessProbe: + tcpSocket: + port: {{ tuple "elasticsearch" "internal" "discovery" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + initialDelaySeconds: 20 + periodSeconds: 10 + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: node.roles + value: "[master]" + - name: HTTP_ENABLE + value: "false" + - name: DISCOVERY_SERVICE + value: {{ tuple "elasticsearch" "discovery" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + - name: ES_JAVA_OPTS + value: "{{ .Values.conf.elasticsearch.env.java_opts.master }}" +{{- if .Values.manifests.certificates }} + - name: JAVA_KEYSTORE_CERT_PATH + value: "/usr/share/elasticsearch/config/ca.crt" + - name: ELASTICSEARCH_PASSWORD + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.elasticsearch.user }} + key: ELASTICSEARCH_PASSWORD +{{- end }} +{{- if .Values.conf.elasticsearch.snapshots.enabled }} +{{- if .Values.manifests.object_bucket_claim }} +{{- include "helm-toolkit.snippets.rgw_s3_bucket_user_env_vars_rook" . | indent 12 }} +{{- else }} +{{- include "helm-toolkit.snippets.rgw_s3_user_env_vars" . | indent 12 }} +{{- end }} +{{- end }} +{{- if .Values.pod.env.master }} +{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.master | indent 12 }} +{{- end }} + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: elasticsearch-logs + mountPath: {{ .Values.conf.elasticsearch.config.path.logs }} + - name: elasticsearch-bin + mountPath: /tmp/elasticsearch.sh + subPath: elasticsearch.sh + readOnly: true + - name: elasticsearch-etc + mountPath: /usr/share/elasticsearch/config/elasticsearch.yml + subPath: elasticsearch.yml + readOnly: true + - name: elasticsearch-etc + mountPath: /usr/share/elasticsearch/config/log4j2.properties + subPath: log4j2.properties + readOnly: true + - name: elasticsearch-etc + mountPath: /usr/share/elasticsearch/config/jvm.options + subPath: jvm.options + readOnly: true + - name: storage + mountPath: {{ .Values.conf.elasticsearch.config.path.data }} +{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.elasticsearch.elasticsearch.internal "path" "/usr/share/elasticsearch/config" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }} +{{ if $mounts_elasticsearch.volumeMounts }}{{ toYaml $mounts_elasticsearch.volumeMounts | indent 12 }}{{ end }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: elasticsearch-logs + emptyDir: {} + - name: elasticsearch-bin + configMap: + name: elasticsearch-bin + defaultMode: 0555 + - name: elasticsearch-etc + secret: + secretName: elasticsearch-etc + defaultMode: 0444 +{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.elasticsearch.elasticsearch.internal | include "helm-toolkit.snippets.tls_volume" | indent 8 }} +{{ if $mounts_elasticsearch.volumes }}{{ toYaml $mounts_elasticsearch.volumes | indent 8 }}{{ end }} +{{- if not .Values.storage.master.enabled }} + - name: storage + emptyDir: {} +{{- else }} + volumeClaimTemplates: + - metadata: + name: storage + spec: + accessModes: {{ .Values.storage.master.pvc.access_mode }} + resources: + requests: + storage: {{ .Values.storage.master.requests.storage }} + storageClassName: {{ .Values.storage.master.storage_class }} +{{- end }} +{{- end }} diff --git a/elasticsearch/values.yaml b/elasticsearch/values.yaml new file mode 100644 index 0000000000..f4bf051ce1 --- /dev/null +++ b/elasticsearch/values.yaml @@ -0,0 +1,993 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for elasticsearch +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +--- +images: + tags: + apache_proxy: docker.io/library/httpd:2.4 + memory_init: docker.io/openstackhelm/heat:wallaby-ubuntu_focal + elasticsearch: docker.io/openstackhelm/elasticsearch-s3:latest-8_9_0 + curator: docker.io/untergeek/curator:8.0.10 + ceph_key_placement: docker.io/openstackhelm/ceph-config-helper:ubuntu_jammy_19.2.1-1-20250207 + s3_bucket: docker.io/openstackhelm/ceph-daemon:ubuntu_jammy_19.2.1-1-20250207 + s3_user: docker.io/openstackhelm/ceph-config-helper:ubuntu_jammy_19.2.1-1-20250207 + helm_tests: docker.io/openstackhelm/heat:wallaby-ubuntu_focal + prometheus_elasticsearch_exporter: quay.io/prometheuscommunity/elasticsearch-exporter:v1.7.0 + dep_check: quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal + snapshot_repository: docker.io/openstackhelm/ceph-config-helper:ubuntu_jammy_19.2.1-1-20250207 + elasticsearch_templates: docker.io/openstackhelm/elasticsearch-s3:latest-8_9_0 + image_repo_sync: docker.io/library/docker:17.07.0 + pull_policy: "IfNotPresent" + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +labels: + client: + node_selector_key: openstack-control-plane + node_selector_value: enabled + data: + node_selector_key: openstack-control-plane + node_selector_value: enabled + exporter: + node_selector_key: openstack-control-plane + node_selector_value: enabled + master: + node_selector_key: openstack-control-plane + node_selector_value: enabled + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + test: + node_selector_key: openstack-control-plane + node_selector_value: enabled + gateway: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - elasticsearch-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + curator: + services: + - endpoint: internal + service: elasticsearch + - endpoint: data + service: elasticsearch + - endpoint: discovery + service: elasticsearch + jobs: + - elasticsearch-register-snapshot-repository + elasticsearch_client: + services: + - endpoint: discovery + service: elasticsearch + jobs: null + elasticsearch_gateway: + services: + - endpoint: discovery + service: elasticsearch + elasticsearch_data: + services: + - endpoint: internal + service: elasticsearch + - endpoint: discovery + service: elasticsearch + jobs: null + elasticsearch_master: + services: null + jobs: null + elasticsearch_templates: + services: + - endpoint: internal + service: elasticsearch + jobs: + - elasticsearch-s3-bucket + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + prometheus_elasticsearch_exporter: + services: + - endpoint: internal + service: elasticsearch + snapshot_repository: + services: + - endpoint: internal + service: elasticsearch + jobs: + - elasticsearch-s3-bucket + verify_repositories: + services: null + jobs: + - create-elasticsearch-templates + s3_user: + services: + - endpoint: internal + service: ceph_object_store + s3_bucket: + jobs: + - elasticsearch-s3-user + tests: + services: null + jobs: + - create-elasticsearch-templates + +pod: + env: + client: null + data: null + master: null + gateway: null + secrets: null + mandatory_access_control: + type: apparmor + elasticsearch-master: + elasticsearch-master: runtime/default + elasticsearch-data: + elasticsearch-data: runtime/default + elasticsearch-client: + elasticsearch-client: runtime/default + elasticsearch-gateway: + elasticsearch-gateway: runtime/default + security_context: + exporter: + pod: + runAsUser: 99 + container: + elasticsearch_exporter: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + client: + pod: + runAsUser: 0 + container: + memory_map_increase: + privileged: true + readOnlyRootFilesystem: true + apache_proxy: + readOnlyRootFilesystem: false + elasticsearch_client: + runAsUser: 1000 + runAsGroup: 1000 + readOnlyRootFilesystem: false + master: + pod: + runAsUser: 0 + container: + memory_map_increase: + privileged: true + readOnlyRootFilesystem: true + elasticsearch_perms: + readOnlyRootFilesystem: true + elasticsearch_master: + runAsUser: 1000 + runAsGroup: 1000 + readOnlyRootFilesystem: false + snapshot_repository: + pod: + runAsUser: 0 + container: + register_snapshot_repository: + readOnlyRootFilesystem: true + test: + pod: + runAsUser: 0 + container: + helm_test: + readOnlyRootFilesystem: true + data: + pod: + runAsUser: 0 + container: + memory_map_increase: + privileged: true + readOnlyRootFilesystem: true + elasticsearch_perms: + readOnlyRootFilesystem: true + elasticsearch_data: + runAsUser: 1000 + runAsGroup: 1000 + # NOTE: This was changed from true to false to account for + # recovery scenarios when the data pods are unexpectedly lost due to + # node outages and shard/index recovery is required + readOnlyRootFilesystem: false + gateway: + pod: + runAsUser: 0 + container: + memory_map_increase: + privileged: true + readOnlyRootFilesystem: true + apache_proxy: + readOnlyRootFilesystem: false + elasticsearch_gateway: + runAsUser: 1000 + runAsGroup: 1000 + readOnlyRootFilesystem: false + curator: + pod: + runAsUser: 0 + container: + curator: + readOnlyRootFilesystem: true + verify_repositories: + pod: + runAsUser: 0 + container: + elasticsearch_verify_repositories: + readOnlyRootFilesystem: true + create_template: + pod: + runAsUser: 0 + container: + create_elasticsearch_template: + readOnlyRootFilesystem: true + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + weight: + default: 10 + replicas: + master: 3 + data: 3 + client: 3 + gateway: 3 + lifecycle: + upgrades: + statefulsets: + pod_replacement_strategy: RollingUpdate + deployments: + revision_history: 3 + pod_replacement_strategy: RollingUpdate + rolling_update: + max_unavailable: 1 + max_surge: 3 + termination_grace_period: + master: + timeout: 600 + data: + timeout: 1200 + client: + timeout: 600 + prometheus_elasticsearch_exporter: + timeout: 600 + probes: + elasticsearch: + elasticsearch-client: + readiness: + enabled: true + params: + initialDelaySeconds: 30 + timeoutSeconds: 30 + liveness: + enabled: true + params: + initialDelaySeconds: 60 + periodSeconds: 10 + mounts: + elasticsearch: + elasticsearch: + elasticsearch_templates: + elasticsearch_templates: + resources: + enabled: false + apache_proxy: + limits: + memory: "1024Mi" + cpu: "2000m" + requests: + memory: "128Mi" + cpu: "100m" + client: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + master: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + data: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + prometheus_elasticsearch_exporter: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + gateway: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + jobs: + curator: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + elasticsearch_templates: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + image_repo_sync: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + snapshot_repository: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + storage_init: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + s3_bucket: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + s3_user: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + tests: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + +network_policy: + elasticsearch: + ingress: + - {} + egress: + - {} + prometheus-elasticsearch-exporter: + ingress: + - {} + egress: + - {} + +secrets: + rgw: + admin: radosgw-s3-admin-creds + elasticsearch: elasticsearch-s3-user-creds + elasticsearch: + user: elasticsearch-user-secrets + oci_image_registry: + elasticsearch: elasticsearch-oci-image-registry-key + tls: + elasticsearch: + elasticsearch: + public: elasticsearch-tls-public + internal: elasticsearch-tls-api + +jobs: + curator: + cron: "* */6 * * *" + history: + success: 3 + failed: 1 + verify_repositories: + cron: "*/30 * * * *" + history: + success: 3 + failed: 1 + create_elasticsearch_templates: + backoffLimit: 6 + +conf: + httpd: | + ServerRoot "/usr/local/apache2" + + Listen 80 + + LoadModule allowmethods_module modules/mod_allowmethods.so + LoadModule mpm_event_module modules/mod_mpm_event.so + LoadModule authn_file_module modules/mod_authn_file.so + LoadModule authn_core_module modules/mod_authn_core.so + LoadModule authz_host_module modules/mod_authz_host.so + LoadModule authz_groupfile_module modules/mod_authz_groupfile.so + LoadModule authz_user_module modules/mod_authz_user.so + LoadModule authz_core_module modules/mod_authz_core.so + LoadModule access_compat_module modules/mod_access_compat.so + LoadModule auth_basic_module modules/mod_auth_basic.so + LoadModule ldap_module modules/mod_ldap.so + LoadModule authnz_ldap_module modules/mod_authnz_ldap.so + LoadModule reqtimeout_module modules/mod_reqtimeout.so + LoadModule filter_module modules/mod_filter.so + LoadModule proxy_html_module modules/mod_proxy_html.so + LoadModule log_config_module modules/mod_log_config.so + LoadModule env_module modules/mod_env.so + LoadModule headers_module modules/mod_headers.so + LoadModule setenvif_module modules/mod_setenvif.so + LoadModule version_module modules/mod_version.so + LoadModule proxy_module modules/mod_proxy.so + LoadModule proxy_connect_module modules/mod_proxy_connect.so + LoadModule proxy_http_module modules/mod_proxy_http.so + LoadModule proxy_balancer_module modules/mod_proxy_balancer.so + LoadModule slotmem_shm_module modules/mod_slotmem_shm.so + LoadModule slotmem_plain_module modules/mod_slotmem_plain.so + LoadModule unixd_module modules/mod_unixd.so + LoadModule status_module modules/mod_status.so + LoadModule autoindex_module modules/mod_autoindex.so + LoadModule rewrite_module modules/mod_rewrite.so + + + User daemon + Group daemon + + + + AllowOverride none + Require all denied + + + + Require all denied + + + ErrorLog /dev/stderr + + LogLevel warn + + + LogFormat "%a %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined + LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" proxy + LogFormat "%h %l %u %t \"%r\" %>s %b" common + + + LogFormat "%a %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combinedio + + + SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded + CustomLog /dev/stdout common + CustomLog /dev/stdout combined + CustomLog /dev/stdout proxy env=forwarded + + + + AllowOverride None + Options None + Require all granted + + + + RequestHeader unset Proxy early + + + + Include conf/extra/proxy-html.conf + + + + + ProxyPass http://localhost:{{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/ + ProxyPassReverse http://localhost:{{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/ + AuthName "Elasticsearch" + AuthType Basic + AuthBasicProvider file ldap + AuthUserFile /usr/local/apache2/conf/.htpasswd + AuthLDAPBindDN {{ .Values.endpoints.ldap.auth.admin.bind }} + AuthLDAPBindPassword {{ .Values.endpoints.ldap.auth.admin.password }} + AuthLDAPURL {{ tuple "ldap" "default" "ldap" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | quote }} + Require valid-user + + + # Restrict access to the Elasticsearch Update By Query API Endpoint to prevent modification of indexed documents + + Require all denied + + # Restrict access to the Elasticsearch Delete By Query API Endpoint to prevent deletion of indexed documents + + Require all denied + + + log4j2: | + status = error + appender.console.type = Console + appender.console.name = console + appender.console.layout.type = PatternLayout + appender.console.layout.pattern = [%d{ISO8601}][%-5p][%-25c{1.}] [%node_name]%marker%m%n + rootLogger.level = info + rootLogger.appenderRef.console.ref = console + jvm_options: | + -Xms1g + -Xmx1g + -Des.networkaddress.cache.ttl=60 + -Des.networkaddress.cache.negative.ttl=10 + -XX:+AlwaysPreTouch + -Xss1m + -Djava.awt.headless=true + -Dfile.encoding=UTF-8 + -Djna.nosys=true + -XX:-OmitStackTraceInFastThrow + -Dio.netty.noUnsafe=true + -Dio.netty.noKeySetOptimization=true + -Dio.netty.recycler.maxCapacityPerThread=0 + -Dlog4j.shutdownHookEnabled=false + -Dlog4j2.disable.jmx=true + -Djava.io.tmpdir=${ES_TMPDIR} + {{- if .Values.manifests.certificates }} + -Djavax.net.ssl.trustStore=/usr/share/elasticsearch/config/elasticsearch-java-truststore + -Djavax.net.ssl.trustStorePassword={{ .Values.endpoints.elasticsearch.auth.admin.password }} + {{- end }} + -XX:+HeapDumpOnOutOfMemoryError + -XX:HeapDumpPath=data + -XX:ErrorFile=logs/hs_err_pid%p.log + 8:-XX:+PrintGCDetails + 8:-XX:+PrintGCDateStamps + 8:-XX:+PrintTenuringDistribution + 8:-XX:+PrintGCApplicationStoppedTime + 8:-Xloggc:logs/gc.log + 8:-XX:+UseGCLogFileRotation + 8:-XX:NumberOfGCLogFiles=32 + 8:-XX:GCLogFileSize=64m + 8-13:-XX:+UseConcMarkSweepGC + 8-13:-XX:CMSInitiatingOccupancyFraction=75 + 8-13:-XX:+UseCMSInitiatingOccupancyOnly + 9-:-Xlog:gc*,gc+age=trace,safepoint:file=logs/gc.log:utctime,pid,tags:filecount=32,filesize=64m + 9-:-Djava.locale.providers=COMPAT + 10-:-XX:UseAVX=2 + init: + max_map_count: 262144 + ceph: + admin_keyring: null + curator: + executable: /curator/curator + action_file: {} + # Remember, leave a key empty if there is no value. None will be a string, + # not a Python "NoneType" + # + # Also remember that all examples have 'disable_action' set to True. If you + # want to use this action as a template, be sure to set this to False after + # copying it. + # + # NOTE(srwilkers): The list of actions below is kept empty, and should be + # driven purely by overrides. As these items are injected as pure YAML, + # the desired configuration should include all fields as to avoid unwanted + # merges with a set of dummy default values. The supplied values can be + # used as an example + # actions: + # 1: + # action: delete_indices + # description: >- + # "Delete indices older than 7 days" + # options: + # timeout_override: + # continue_if_exception: False + # ignore_empty_list: True + # disable_action: True + # filters: + # - filtertype: pattern + # kind: prefix + # value: logstash- + # - filtertype: age + # source: name + # direction: older + # timestring: '%Y.%m.%d' + # unit: days + # unit_count: 7 + # 2: + # action: delete_indices + # description: >- + # "Delete indices by age if available disk space is + # less than 80% total disk" + # options: + # timeout_override: 600 + # continue_if_exception: False + # ignore_empty_list: True + # disable_action: True + # filters: + # - filtertype: pattern + # kind: prefix + # value: logstash- + # - filtertype: space + # source: creation_date + # use_age: True + # # This space assumes the default PVC size of 5Gi times three data + # # replicas. This must be adjusted if changed due to Curator being + # # unable to calculate percentages of total disk space + # disk_space: 12 + # 3: + # action: snapshot + # description: >- + # "Snapshot indices older than one day" + # options: + # repository: logstash_snapshots + # # Leaving this blank results in the default name format + # name: + # wait_for_completion: True + # max_wait: 3600 + # wait_interval: 10 + # timeout_override: 600 + # ignore_empty_list: True + # continue_if_exception: False + # disable_action: True + # filters: + # - filtertype: age + # source: name + # direction: older + # timestring: '%Y.%m.%d' + # unit: days + # unit_count: 1 + # 4: + # action: delete_snapshots + # description: >- + # "Delete snapshots older than 30 days" + # options: + # repository: logstash_snapshots + # disable_action: True + # timeout_override: 600 + # ignore_empty_list: True + # filters: + # - filtertype: pattern + # kind: prefix + # value: curator- + # exclude: + # - filtertype: age + # source: creation_date + # direction: older + # unit: days + # unit_count: 30 + config: + # Remember, leave a key empty if there is no value. None will be a string, + # not a Python "NoneType" + elasticsearch: + client: + hosts: ${ELASTICSEARCH_URL} + request_timeout: 60 + other_settings: + username: ${ELASTICSEARCH_USERNAME} + password: ${ELASTICSEARCH_PASSWORD} + + logging: + loglevel: INFO + logformat: json + blacklist: ['elastic_transport', 'urllib3'] + elasticsearch: + config: + xpack: + security: + enabled: false + bootstrap: + # As far as we run the pod as non-root, we can't make locking memory unlimited. + # configure the memory locking limits on host itself of disable swap completely. + memory_lock: false + cluster: + name: elasticsearch + discovery: + # NOTE(srwilkers): This gets configured dynamically via endpoint lookups + seed_hosts: null + network: + host: 0.0.0.0 + s3: + client: {} + path: + data: /data + logs: /logs + snapshots: + enabled: false + env: + java_opts: + client: "-Xms256m -Xmx256m" + data: "-Xms256m -Xmx256m" + master: "-Xms256m -Xmx256m" + prometheus_elasticsearch_exporter: + es: + timeout: 30s + all: true + indices: true + indices_settings: true + indices_mappings: true + aliases: false + shards: true + snapshots: true + cluster_settings: true + slm: true + data_stream: false + log: + format: logfmt + level: info + + api_objects: {} + # Fill this map with API objects to create once Elasticsearch is deployed + # name: # This name can be completely arbitrary + # method: # Defaults to PUT + # endpoint: # Path for the request + # body: # Body of the request in yaml (Converted to Json in Template) + # Example: ILM Policy + # ilm_policy: + # endpoint: _ilm/policy/delete_all_indexes + # body: + # policy: + # phases: + # delete: + # min_age: 14d + # actions: + # delete: {} + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + elasticsearch: + username: elasticsearch + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + elasticsearch: + name: elasticsearch + namespace: null + auth: + admin: + username: admin + password: changeme + logging: + username: remote + password: changeme + hosts: + data: elasticsearch-data + default: elasticsearch-logging + discovery: elasticsearch-discovery + gateway: elasticsaerch-gateway + public: elasticsearch + host_fqdn_override: + default: null + path: + default: null + scheme: + default: http + gateway: tcp + port: + client: + default: 9200 + http: + default: 80 + discovery: + default: 9300 + prometheus_elasticsearch_exporter: + namespace: null + hosts: + default: elasticsearch-exporter + host_fqdn_override: + default: null + path: + default: /metrics + scheme: + default: 'http' + port: + metrics: + default: 9108 + ldap: + hosts: + default: ldap + auth: + admin: + bind: "cn=admin,dc=cluster,dc=local" + password: password + host_fqdn_override: + default: null + path: + default: "/ou=People,dc=cluster,dc=local" + scheme: + default: ldap + port: + ldap: + default: 389 + ceph_object_store: + name: radosgw + namespace: null + hosts: + default: ceph-rgw + public: radosgw + host_fqdn_override: + default: null + path: + default: null + scheme: + default: http + port: + api: + default: 8088 + public: 80 + +monitoring: + prometheus: + enabled: false + elasticsearch_exporter: + scrape: true + +network: + elasticsearch: + ingress: + public: true + classes: + namespace: "nginx" + cluster: "nginx-cluster" + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / + node_port: + enabled: false + port: 30920 + remote_clustering: + enabled: false + node_port: + port: 30930 + +storage: + data: + enabled: true + pvc: + name: pvc-elastic + access_mode: ["ReadWriteOnce"] + requests: + storage: 5Gi + storage_class: general + master: + enabled: true + pvc: + name: pvc-elastic + access_mode: ["ReadWriteOnce"] + requests: + storage: 1Gi + storage_class: general + s3: + clients: {} + # These values configure the s3 clients section of elasticsearch.yml + # See: https://www.elastic.co/guide/en/elasticsearch/plugins/current/repository-s3-client.html + # default: + # auth: + # # Values under auth are written to the Secret $client-s3-user-secret + # # and the access & secret keys are added to the elasticsearch keystore + # username: elasticsearch + # access_key: "elastic_access_key" + # secret_key: "elastic_secret_key" + # settings: + # # Configure Client Settings here (https://www.elastic.co/guide/en/elasticsearch/plugins/current/repository-s3-client.html) + # # endpoint: Defaults to the ceph-rgw endpoint + # # protocol: Defaults to http + # path_style_access: true # Required for ceph-rgw S3 API + # create_user: true # Attempt to create the user at the ceph_object_store endpoint, authenticating using the secret named at .Values.secrets.rgw.admin + # backup: + # auth: + # username: elasticsearch + # access_key: "backup_access_key" + # secret_key: "backup_secret_key" + # settings: + # endpoint: s3.example.com # Specify your own s3 endpoint (defaults to the ceph_object_store endpoint) + # path_style_access: false + # create_user: false + buckets: {} + # List of buckets to create (if required). + # (The client field references one of the clients defined above) + # - name: elasticsearch-bucket + # client: default + # options: # list of extra options for s3cmd + # - --region="default:osh-infra" + # # SSL connection option for s3cmd + # ssl_connecton_option: --ca-certs={path to mounted ca.crt} + # - name: backup-bucket + # client: backup + # options: # list of extra options for s3cmd + # - --region="default:backup" + # # SSL connection option for s3cmd + # ssl_connecton_option: --ca-certs={path to mounted ca.crt} + +manifests: + certificates: false + configmap_bin_curator: false + configmap_bin_elasticsearch: true + configmap_etc_curator: false + configmap_etc_elasticsearch: true + configmap_etc_templates: true + cron_curator: false + cron_verify_repositories: true + deployment_client: true + ingress: true + job_elasticsearch_templates: true + job_image_repo_sync: true + job_snapshot_repository: true + job_s3_user: true + job_s3_bucket: true + helm_tests: true + secret_elasticsearch: true + secret_s3: true + monitoring: + prometheus: + configmap_bin_exporter: true + deployment_exporter: true + network_policy_exporter: false + service_exporter: true + network_policy: false + secret_ingress_tls: true + secret_registry: true + service_data: true + service_discovery: true + service_ingress: true + service_logging: true + statefulset_data: true + statefulset_master: true + object_bucket_claim: false +... diff --git a/etcd/Chart.yaml b/etcd/Chart.yaml new file mode 100644 index 0000000000..40f709dc1d --- /dev/null +++ b/etcd/Chart.yaml @@ -0,0 +1,30 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: v3.4.3 +description: OpenStack-Helm etcd +name: etcd +version: 2024.2.0 +home: https://coreos.com/etcd/ +icon: https://raw.githubusercontent.com/CloudCoreo/etcd-cluster/master/images/icon.png +sources: + - https://github.com/coreos/etcd/ + - https://opendev.org/openstack/openstack-helm +maintainers: + - name: OpenStack-Helm Authors +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit + version: ">= 0.1.0" +... diff --git a/etcd/templates/bin/_etcd-db-compact.sh.tpl b/etcd/templates/bin/_etcd-db-compact.sh.tpl new file mode 100644 index 0000000000..ff6af04f4a --- /dev/null +++ b/etcd/templates/bin/_etcd-db-compact.sh.tpl @@ -0,0 +1,47 @@ +#!/bin/sh + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -x + +export ETCDCTL_API=3 + +{{- if .Values.jobs.db_compact.command_timeout }} +COMMAND_TIMEOUT='--command-timeout={{ .Values.jobs.db_compact.command_timeout }}' +{{- else }} +COMMAND_TIMEOUT='' +{{- end }} + +ENDPOINTS=$(etcdctl member list --endpoints=http://${ETCD_SERVICE_HOST}:${ETCD_SERVICE_PORT} ${COMMAND_TIMEOUT}| cut -d, -f5 | sed -e 's/ //g' | paste -sd ',') + +etcdctl --endpoints=${ENDPOINTS} endpoint status --write-out="table" ${COMMAND_TIMEOUT} + +rev=$(etcdctl --endpoints=http://${ETCD_SERVICE_HOST}:${ETCD_SERVICE_PORT} endpoint status --write-out="json" ${COMMAND_TIMEOUT}| egrep -o '"revision":[0-9]*' | egrep -o '[0-9].*') +compact_result=$(etcdctl compact --physical=true --endpoints=${ENDPOINTS} $rev ${COMMAND_TIMEOUT} 2>&1 > /dev/null) +compact_res=$? + +if [[ $compact_res -ne 0 ]]; then + match_pattern=$(echo ${compact_result} | egrep '(mvcc: required revision has been compacted.*$)') + match_pattern_res=$? + if [[ $match_pattern_res -eq 0 ]]; then + exit 0 + else + echo "Failed to compact database: $compact_result" + exit $compact_res + fi +else + etcdctl defrag --endpoints=${ENDPOINTS} ${COMMAND_TIMEOUT} + etcdctl --endpoints=${ENDPOINTS} endpoint status --write-out="table" ${COMMAND_TIMEOUT} +fi diff --git a/etcd/templates/bin/_etcd-healthcheck.sh.tpl b/etcd/templates/bin/_etcd-healthcheck.sh.tpl new file mode 100644 index 0000000000..ef5442df54 --- /dev/null +++ b/etcd/templates/bin/_etcd-healthcheck.sh.tpl @@ -0,0 +1,24 @@ +#!/bin/sh + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -x + +export ETCDCTL_API=3 + +ETCD_CLIENT_PORT={{ tuple "etcd" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +DISCOVERY_DOMAIN={{ tuple "etcd" "discovery" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }} + +etcdctl endpoint health --endpoints=${POD_NAME}.${DISCOVERY_DOMAIN}:${ETCD_CLIENT_PORT} diff --git a/etcd/templates/bin/_etcd.sh.tpl b/etcd/templates/bin/_etcd.sh.tpl new file mode 100644 index 0000000000..3ac97648d0 --- /dev/null +++ b/etcd/templates/bin/_etcd.sh.tpl @@ -0,0 +1,75 @@ +#!/bin/sh + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +active_members_present() { + res=1 + for endpoint in $(echo $ETCD_ENDPOINTS | tr ',' '\n'); do + if etcdctl endpoint health --endpoints=$endpoint >/dev/null 2>&1; then + res=$? + if [[ "$res" == 0 ]]; then + break + fi + fi + done + echo $res +} + +ETCD_REPLICAS={{ .Values.pod.replicas.etcd }} +PEER_PREFIX_NAME={{- printf "%s-%s" .Release.Name "etcd" }} +DISCOVERY_DOMAIN={{ tuple "etcd" "discovery" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }} +ETCD_PEER_PORT=2380 +ETCD_CLIENT_PORT={{ tuple "etcd" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +ETCD_PROTOCOL={{ tuple "etcd" "internal" "client" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" }} +PEERS="${PEER_PREFIX_NAME}-0=${ETCD_PROTOCOL}://${PEER_PREFIX_NAME}-0.${DISCOVERY_DOMAIN}:${ETCD_PEER_PORT}" +ETCD_ENDPOINTS="${ETCD_PROTOCOL}://${PEER_PREFIX_NAME}-0.${DISCOVERY_DOMAIN}:${ETCD_PEER_PORT}" +if [[ ${ETCD_REPLICAS} -gt 1 ]] ; then + for i in $(seq 1 $(( ETCD_REPLICAS - 1 ))); do + PEERS="$PEERS,${PEER_PREFIX_NAME}-${i}=${ETCD_PROTOCOL}://${PEER_PREFIX_NAME}-${i}.${DISCOVERY_DOMAIN}:${ETCD_PEER_PORT}" + ETCD_ENDPOINTS="${ETCD_ENDPOINTS},${ETCD_PROTOCOL}://${PEER_PREFIX_NAME}-${i}.${DISCOVERY_DOMAIN}:${ETCD_PEER_PORT}" + done +fi +ADVERTISE_PEER_URL="${ETCD_PROTOCOL}://${HOSTNAME}.${DISCOVERY_DOMAIN}:${ETCD_PEER_PORT}" +ADVERTISE_CLIENT_URL="${ETCD_PROTOCOL}://${HOSTNAME}.${DISCOVERY_DOMAIN}:${ETCD_CLIENT_PORT}" + +ETCD_INITIAL_CLUSTER_STATE=new + +if [[ -z "$(ls -A $ETCD_DATA_DIR)" ]]; then + echo "State directory $ETCD_DATA_DIR is empty." + if [[ $(active_members_present) -eq 0 ]]; then + ETCD_INITIAL_CLUSTER_STATE=existing + member_id=$(etcdctl --endpoints=${ETCD_ENDPOINTS} member list | grep -w ${ADVERTISE_CLIENT_URL} | awk -F "," '{ print $1 }') + if [[ -n "$member_id" ]]; then + echo "Current node is a member of cluster, member_id: ${member_id}" + echo "Rejoining..." + echo "Removing member from the cluster" + etcdctl member remove "$member_id" --endpoints=${ETCD_ENDPOINTS} + etcdctl member add ${ADVERTISE_CLIENT_URL} --peer-urls=${ADVERTISE_PEER_URL} --endpoints=${ETCD_ENDPOINTS} + fi + else + echo "Do not have active members. Starting initial cluster state." + fi +fi + +exec etcd \ + --name ${HOSTNAME} \ + --listen-peer-urls ${ETCD_PROTOCOL}://0.0.0.0:${ETCD_PEER_PORT} \ + --listen-client-urls ${ETCD_PROTOCOL}://0.0.0.0:${ETCD_CLIENT_PORT} \ + --advertise-client-urls ${ADVERTISE_CLIENT_URL} \ + --initial-advertise-peer-urls ${ADVERTISE_PEER_URL} \ + --initial-cluster ${PEERS} \ + --initial-cluster-state ${ETCD_INITIAL_CLUSTER_STATE} diff --git a/etcd/templates/configmap-bin.yaml b/etcd/templates/configmap-bin.yaml new file mode 100644 index 0000000000..d5407333b9 --- /dev/null +++ b/etcd/templates/configmap-bin.yaml @@ -0,0 +1,36 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_bin }} +{{- $envAll := . }} +{{- $configMapBinName := printf "%s-%s" $envAll.Release.Name "etcd-bin" }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $configMapBinName }} +data: +{{- if .Values.images.local_registry.active }} + image-repo-sync.sh: | +{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} +{{- end }} + etcd.sh: | +{{ tuple "bin/_etcd.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- if .Values.manifests.cron_job_db_compact }} + etcd-db-compact.sh: | +{{ tuple "bin/_etcd-db-compact.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} + etcd-healthcheck.sh: | +{{ tuple "bin/_etcd-healthcheck.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} diff --git a/etcd/templates/cron-job-db-compact.yaml b/etcd/templates/cron-job-db-compact.yaml new file mode 100644 index 0000000000..80a64e11cb --- /dev/null +++ b/etcd/templates/cron-job-db-compact.yaml @@ -0,0 +1,75 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.cron_job_db_compact }} +{{- $envAll := . }} + +{{- $configMapBinName := printf "%s-%s" $envAll.Release.Name "etcd-bin" }} + +{{- $serviceAccountName := "etcd-db-compact" }} +{{ tuple $envAll "db_compact" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: etcd-db-compaction + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +spec: + schedule: {{ .Values.jobs.db_compact.cron | quote }} + successfulJobsHistoryLimit: {{ .Values.jobs.db_compact.history.success }} + failedJobsHistoryLimit: {{ .Values.jobs.db_compact.history.failed }} + {{- if .Values.jobs.db_compact.starting_deadline }} + startingDeadlineSeconds: {{ .Values.jobs.db_compact.starting_deadline }} + {{- end }} + concurrencyPolicy: Forbid + jobTemplate: + metadata: + labels: +{{ tuple $envAll "etcd" "db-compact" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + spec: + template: + metadata: + labels: +{{ tuple $envAll "etcd" "db-compact" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 12 }} + spec: +{{ dict "envAll" $envAll "application" "etcd_db_compact" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 10 }} + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} + initContainers: +{{ tuple $envAll "db_compact" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 12 }} + containers: + - name: etcd-db-compact +{{ tuple $envAll "etcd_db_compact" | include "helm-toolkit.snippets.image" | indent 14 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.db_compact | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }} +{{ dict "envAll" $envAll "application" "etcd_db_compact" "container" "etcd_db_compact" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 14 }} + command: + - /tmp/etcd-db-compact.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: etcd-bin + mountPath: /tmp/etcd-db-compact.sh + subPath: etcd-db-compact.sh + readOnly: true + volumes: + - name: pod-tmp + emptyDir: {} + - name: etcd-bin + configMap: + name: {{ $configMapBinName | quote }} + defaultMode: 0555 +{{- end }} diff --git a/etcd/templates/job-image-repo-sync.yaml b/etcd/templates/job-image-repo-sync.yaml new file mode 100644 index 0000000000..07433b0ead --- /dev/null +++ b/etcd/templates/job-image-repo-sync.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "etcd" -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/etcd/templates/secret-registry.yaml b/etcd/templates/secret-registry.yaml new file mode 100644 index 0000000000..da979b3223 --- /dev/null +++ b/etcd/templates/secret-registry.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_registry .Values.endpoints.oci_image_registry.auth.enabled }} +{{ include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) }} +{{- end }} diff --git a/etcd/templates/service-discovery.yaml b/etcd/templates/service-discovery.yaml new file mode 100644 index 0000000000..83a0808d93 --- /dev/null +++ b/etcd/templates/service-discovery.yaml @@ -0,0 +1,34 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- if .Values.manifests.service_discovery }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "etcd" "discovery" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +spec: + ports: + - name: client + port: {{ tuple "etcd" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + protocol: TCP + targetPort: client + - name: peer + port: {{ tuple "etcd_discovery" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + protocol: TCP + targetPort: peer + publishNotReadyAddresses: true + clusterIP: None + selector: +{{ tuple $envAll "etcd" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- end }} diff --git a/etcd/templates/service.yaml b/etcd/templates/service.yaml new file mode 100644 index 0000000000..7c6dcf8a9b --- /dev/null +++ b/etcd/templates/service.yaml @@ -0,0 +1,33 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- if .Values.manifests.service }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "etcd" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +spec: + sessionAffinity: ClientIP + ports: + - name: client + port: {{ tuple "etcd" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + protocol: TCP + targetPort: client + - name: peer + port: {{ tuple "etcd_discovery" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + protocol: TCP + targetPort: peer + selector: +{{ tuple $envAll "etcd" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- end }} diff --git a/etcd/templates/statefulset.yaml b/etcd/templates/statefulset.yaml new file mode 100644 index 0000000000..c6008167a2 --- /dev/null +++ b/etcd/templates/statefulset.yaml @@ -0,0 +1,118 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- define "etcdProbeTemplate" }} +exec: + command: + - /tmp/etcd-healthcheck.sh +{{- end }} + +{{- if .Values.manifests.statefulset }} +{{- $envAll := . }} + +{{- $rcControllerName := printf "%s-%s" $envAll.Release.Name "etcd" }} +{{- $configMapBinName := printf "%s-%s" $envAll.Release.Name "etcd-bin" }} + +{{ tuple $envAll "etcd" $rcControllerName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ $rcControllerName | quote }} + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "etcd" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + podManagementPolicy: "Parallel" + serviceName: "{{ tuple "etcd" "discovery" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}" + replicas: {{ .Values.pod.replicas.etcd }} + selector: + matchLabels: +{{ tuple $envAll "etcd" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} + template: + metadata: + labels: +{{ tuple $envAll "etcd" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} +{{ dict "envAll" $envAll "podName" "etcd" "containerNames" (list "init" "etcd") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + spec: + serviceAccountName: {{ $rcControllerName | quote }} +{{ dict "envAll" $envAll "application" "etcd" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + affinity: +{{ tuple $envAll "etcd" "server" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} + nodeSelector: + {{ .Values.labels.server.node_selector_key }}: {{ .Values.labels.server.node_selector_value }} + initContainers: +{{ tuple $envAll "etcd" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: etcd +{{ tuple $envAll "etcd" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ dict "envAll" $envAll "application" "etcd" "container" "etcd" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} +{{ dict "envAll" . "component" "etcd" "container" "etcd" "type" "readiness" "probeTemplate" (include "etcdProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }} +{{ dict "envAll" . "component" "etcd" "container" "etcd" "type" "liveness" "probeTemplate" (include "etcdProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }} + env: +{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.etcd | indent 12 }} + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + command: + - /tmp/etcd.sh + ports: + - containerPort: {{ tuple "etcd" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + name: client + protocol: TCP + - containerPort: {{ tuple "etcd_discovery" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + name: peer + protocol: TCP + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: etcd-bin + mountPath: /tmp/etcd.sh + subPath: etcd.sh + readOnly: true + - name: etcd-data + mountPath: /var/lib/etcd + - name: etcd-bin + mountPath: /tmp/etcd-healthcheck.sh + subPath: etcd-healthcheck.sh + readOnly: true + volumes: + - name: pod-tmp + emptyDir: {} + - name: etcd-bin + configMap: + name: {{ $configMapBinName | quote }} + defaultMode: 0555 + {{- if not .Values.volume.enabled }} + - name: etcd-data + emptyDir: {} + {{- end }} +{{- end }} +{{- if .Values.volume.enabled }} + volumeClaimTemplates: + - metadata: + name: etcd-data + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: {{ .Values.volume.size }} + {{- if ne .Values.volume.class_name "default" }} + storageClassName: {{ .Values.volume.class_name }} + {{- end }} +{{- end }} diff --git a/etcd/values.yaml b/etcd/values.yaml new file mode 100644 index 0000000000..effaa7a6d0 --- /dev/null +++ b/etcd/values.yaml @@ -0,0 +1,222 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for etcd. +# This is a YAML-formatted file. +# Declare name/value pairs to be passed into your templates. +# name: value + +--- +images: + tags: + etcd: 'registry.k8s.io/etcd-amd64:3.4.3' + dep_check: quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal + image_repo_sync: docker.io/library/docker:17.07.0 + etcd_db_compact: 'registry.k8s.io/etcd-amd64:3.4.3' + pull_policy: "IfNotPresent" + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +labels: + server: + node_selector_key: openstack-control-plane + node_selector_value: enabled + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - etcd-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + etcd: + jobs: null + db_compact: + services: + - endpoint: internal + service: etcd + +pod: + env: + etcd: + ETCD_DATA_DIR: /var/lib/etcd + ETCD_INITIAL_CLUSTER_TOKEN: etcd-cluster-1 + security_context: + etcd: + pod: + runAsUser: 65534 + container: + etcd: + runAsUser: 0 + readOnlyRootFilesystem: false + etcd_db_compact: + pod: + runAsUser: 65534 + runAsNonRoot: true + allowPrivilegeEscalation: false + container: + etcd_db_compact: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + mandatory_access_control: + type: apparmor + etcd: + init: runtime/default + etcd: runtime/default + probes: + etcd: + etcd: + readiness: + enabled: True + params: + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 1 + liveness: + enabled: True + params: + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 1 + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + weight: + default: 10 + replicas: + etcd: 1 + lifecycle: + upgrades: + deployments: + pod_replacement_strategy: RollingUpdate + revision_history: 3 + rolling_update: + max_surge: 3 + max_unavailable: 1 + resources: + jobs: + image_repo_sync: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + db_compact: + requests: + memory: "128Mi" + cpu: "100m" + +secrets: + oci_image_registry: + etcd: etcd-oci-image-registry-key + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + etcd: + username: etcd + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + etcd: + name: etcd + hosts: + default: etcd + discovery: etcd-discovery + host_fqdn_override: + default: null + path: + default: null + scheme: + default: 'http' + port: + client: + default: 2379 + etcd_discovery: + name: etcd-discovery + hosts: + default: etcd-discovery + host_fqdn_override: + default: null + path: + default: null + scheme: + default: 'http' + port: + client: + default: 2380 + +volume: + enabled: false + class_name: general + size: 5Gi + +jobs: + db_compact: + cron: "1 */2 * * *" + starting_deadline: 600 + # Timeout have to be set the same format + # as it is for etcdctl 120s, 1m etc. + command_timeout: 120s + history: + success: 3 + failed: 1 + +manifests: + configmap_bin: true + statefulset: true + job_image_repo_sync: true + secret_registry: true + service: true + service_discovery: true + cron_job_db_compact: false +... diff --git a/falco/Chart.yaml b/falco/Chart.yaml new file mode 100644 index 0000000000..48b37fb15a --- /dev/null +++ b/falco/Chart.yaml @@ -0,0 +1,36 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +name: falco +version: 2024.2.0 +appVersion: 0.11.1 +description: Sysdig Falco +keywords: + - monitoring + - security + - alerting + - metric + - troubleshooting + - run-time +home: https://www.sysdig.com/opensource/falco/ +icon: https://sysdig.com/wp-content/uploads/2016/08/falco_blog_480.jpg +sources: + - https://github.com/draios/falco +maintainers: + - name: OpenStack-Helm Authors +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit + version: ">= 0.1.0" +... diff --git a/falco/templates/bin/_falco.sh.tpl b/falco/templates/bin/_falco.sh.tpl new file mode 100644 index 0000000000..d1ec7bec6a --- /dev/null +++ b/falco/templates/bin/_falco.sh.tpl @@ -0,0 +1,18 @@ +#!/bin/sh +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +exec /usr/bin/falco -K /var/run/secrets/kubernetes.io/serviceaccount/token -k https://kubernetes.default -pk diff --git a/falco/templates/configmap-bin.yaml b/falco/templates/configmap-bin.yaml new file mode 100644 index 0000000000..4950bcb026 --- /dev/null +++ b/falco/templates/configmap-bin.yaml @@ -0,0 +1,25 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_bin }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: falco-bin +data: + falco.sh: | +{{ tuple "bin/_falco.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} diff --git a/falco/templates/configmap-etc.yaml b/falco/templates/configmap-etc.yaml new file mode 100644 index 0000000000..ae23e6d414 --- /dev/null +++ b/falco/templates/configmap-etc.yaml @@ -0,0 +1,26 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_etc }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: falco +data: + falco.yaml: {{ toYaml .Values.conf.config | b64enc }} + falco_rules.yaml: {{ .Values.conf.rules.falco_rules | b64enc }} + falco_rules.local.yaml: {{ .Values.conf.rules.falco_rules_local | b64enc }} +{{- end }} diff --git a/falco/templates/configmap-rules.yaml b/falco/templates/configmap-rules.yaml new file mode 100644 index 0000000000..ab208cd204 --- /dev/null +++ b/falco/templates/configmap-rules.yaml @@ -0,0 +1,24 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.conf.rules.custom_rules .Values.manifests.configmap_custom_rules }} +apiVersion: v1 +kind: Secret +metadata: + name: falco-rules +data: +{{- range $file, $content := .Values.conf.rules.custom_rules }} + {{ $file }}: {{ $content | b64enc }} +{{- end }} +{{- end }} diff --git a/falco/templates/daemonset.yaml b/falco/templates/daemonset.yaml new file mode 100644 index 0000000000..dbb0df31c7 --- /dev/null +++ b/falco/templates/daemonset.yaml @@ -0,0 +1,149 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.daemonset }} +{{- $envAll := . }} + +{{- $serviceAccountName := "falcon-service" }} +{{ tuple $envAll "falco" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ $serviceAccountName }} +rules: + - apiGroups: + - "" + resources: + - nodes + - namespaces + - pods + - replicationcontrollers + - services + - events + - configmaps + verbs: + - get + - list + - watch + - nonResourceURLs: + - /healthz + - /healthz/* + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ $serviceAccountName }} +roleRef: + kind: ClusterRole + name: {{ $serviceAccountName }} + apiGroup: rbac.authorization.k8s.io +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: falco-agent + labels: +{{ tuple $envAll "falco" "falco-agent" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + selector: + matchLabels: +{{ tuple $envAll "falco" "falco-agent" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} +{{ tuple $envAll "falco" | include "helm-toolkit.snippets.kubernetes_upgrades_daemonset" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "falco" "falco-agent" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} + spec: + serviceAccountName: {{ $serviceAccountName }} +{{ tuple $envAll "falco" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }} + containers: + - name: falco +{{ tuple $envAll "falco" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.falco | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + securityContext: + privileged: true + args: + - /tmp/falco.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - mountPath: /tmp/falco.sh + name: falco-bin + subPath: falco.sh + readOnly: true + - mountPath: /host/dev + name: dev-fs + - mountPath: /host/proc + name: proc-fs + readOnly: true + - mountPath: /host/boot + name: boot-fs + readOnly: true + - mountPath: /host/lib/modules + name: lib-modules + readOnly: true + - mountPath: /host/usr + name: usr-fs + readOnly: true + - mountPath: /etc/falco + name: config-volume + {{- if .Values.conf.rules.custom_rules }} + - mountPath: /etc/falco/rules.d + name: rules-volume + {{- end }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: falco-bin + configMap: + name: falco-bin + defaultMode: 0555 + - name: dshm + emptyDir: + medium: Memory + - name: dev-fs + hostPath: + path: /dev + - name: proc-fs + hostPath: + path: /proc + - name: boot-fs + hostPath: + path: /boot + - name: lib-modules + hostPath: + path: /lib/modules + - name: usr-fs + hostPath: + path: /usr + - name: config-volume + secret: + secretName: falco + {{- if .Values.conf.rules.custom_rules }} + - name: rules-volume + secret: + secretName: falco-rules + {{- end }} +{{- end }} diff --git a/falco/templates/job-image-repo-sync.yaml b/falco/templates/job-image-repo-sync.yaml new file mode 100644 index 0000000000..e6adca13af --- /dev/null +++ b/falco/templates/job-image-repo-sync.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "falco" -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/falco/templates/secret-registry.yaml b/falco/templates/secret-registry.yaml new file mode 100644 index 0000000000..da979b3223 --- /dev/null +++ b/falco/templates/secret-registry.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_registry .Values.endpoints.oci_image_registry.auth.enabled }} +{{ include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) }} +{{- end }} diff --git a/falco/values.yaml b/falco/values.yaml new file mode 100644 index 0000000000..929a6bce69 --- /dev/null +++ b/falco/values.yaml @@ -0,0 +1,1388 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + pull_policy: IfNotPresent + tags: + dep_check: quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal + falco: docker.io/sysdig/falco:0.12.1 + image_repo_sync: docker.io/library/docker:17.07.0 + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +secrets: + oci_image_registry: + falco: falco-oci-image-registry-key + +endpoints: + cluster_domain_suffix: cluster.local + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + falco: + username: falco + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + +pod: + resources: + enabled: false + falco: + requests: + memory: "128Mi" + cpu: "20m" + limits: + memory: "128Mi" + cpu: "30m" + jobs: + image_repo_sync: + requests: + memory: "128Mi" + cpu: "20m" + limits: + memory: "128Mi" + cpu: "30m" + lifecycle: + upgrades: + daemonsets: + pod_replacement_strategy: RollingUpdate + falco: + enabled: true + min_ready_seconds: 0 + max_unavailable: 1 + tolerations: + falco: + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + + +conf: + config: + # The location of the rules file(s). This can contain one or more paths to + # separate rules files. + rules_file: + - /etc/falco/falco_rules.yaml + - /etc/falco/falco_rules.local.yaml + - /etc/falco/rules.d + + # Whether to output events in json or text + json_output: false + + # When using json output, whether or not to include the "output" property + # itself (e.g. "File below a known binary directory opened for writing + # (user=root ....") in the json output. + json_include_output_property: true + + # Send information logs to stderr and/or syslog Note these are *not* security + # notification logs! These are just Falco lifecycle (and possibly error) logs. + log_stderr: true + log_syslog: true + + # Minimum log level to include in logs. Note: these levels are + # separate from the priority field of rules. This refers only to the + # log level of falco's internal logging. Can be one of "emergency", + # "alert", "critical", "error", "warning", "notice", "info", "debug". + log_level: info + + # Minimum rule priority level to load and run. All rules having a + # priority more severe than this level will be loaded/run. Can be one + # of "emergency", "alert", "critical", "error", "warning", "notice", + # "info", "debug". + priority: debug + + # Whether or not output to any of the output channels below is + # buffered. + buffered_outputs: false + + # A throttling mechanism implemented as a token bucket limits the + # rate of falco notifications. This throttling is controlled by the following configuration + # options: + # - rate: the number of tokens (i.e. right to send a notification) + # gained per second. Defaults to 1. + # - max_burst: the maximum number of tokens outstanding. Defaults to 1000. + # + # With these defaults, falco could send up to 1000 notifications after + # an initial quiet period, and then up to 1 notification per second + # afterward. It would gain the full burst back after 1000 seconds of + # no activity. + outputs: + rate: 1 + max_burst: 1000 + + # Where security notifications should go. + # Multiple outputs can be enabled. + syslog_output: + enabled: true + + # If keep_alive is set to true, the file will be opened once and + # continuously written to, with each output message on its own + # line. If keep_alive is set to false, the file will be re-opened + # for each output message. + # + # Also, the file will be closed and reopened if falco is signaled with + # SIGUSR1. + file_output: + enabled: false + keep_alive: false + filename: ./events.txt + + stdout_output: + enabled: true + + # Possible additional things you might want to do with program output: + # - send to a slack webhook: + # program: "jq '{text: .output}' | curl -d @- -X POST https://hooks.slack.com/services/XXX" + # - logging (alternate method than syslog): + # program: logger -t falco-test + # - send over a network connection: + # program: nc host.example.com 80 + + # If keep_alive is set to true, the program will be started once and + # continuously written to, with each output message on its own + # line. If keep_alive is set to false, the program will be re-spawned + # for each output message. + # + # Also, the program will be closed and reopened if falco is signaled with + # SIGUSR1. + program_output: + enabled: false + keep_alive: false + program: mail -s "Falco Notification" someone@example.com + rules: + falco_rules: | + - macro: open_write + condition: (evt.type=open or evt.type=openat) and evt.is_open_write=true and fd.typechar='f' and fd.num>=0 + - macro: open_read + condition: (evt.type=open or evt.type=openat) and evt.is_open_read=true and fd.typechar='f' and fd.num>=0 + - macro: never_true + condition: (evt.num=0) + - macro: always_true + condition: (evt.num=>0) + - macro: proc_name_exists + condition: (proc.name!="") + - macro: rename + condition: evt.type in (rename, renameat) + - macro: mkdir + condition: evt.type = mkdir + - macro: remove + condition: evt.type in (rmdir, unlink, unlinkat) + - macro: modify + condition: rename or remove + - macro: spawned_process + condition: evt.type = execve and evt.dir=< + - macro: bin_dir + condition: fd.directory in (/bin, /sbin, /usr/bin, /usr/sbin) + - macro: bin_dir_mkdir + condition: > + (evt.arg[1] startswith /bin/ or + evt.arg[1] startswith /sbin/ or + evt.arg[1] startswith /usr/bin/ or + evt.arg[1] startswith /usr/sbin/) + - macro: bin_dir_rename + condition: > + evt.arg[1] startswith /bin/ or + evt.arg[1] startswith /sbin/ or + evt.arg[1] startswith /usr/bin/ or + evt.arg[1] startswith /usr/sbin/ + - macro: etc_dir + condition: fd.name startswith /etc/ + - macro: root_dir + condition: ((fd.directory=/ or fd.name startswith /root) and fd.name contains "/") + - list: shell_binaries + items: [bash, csh, ksh, sh, tcsh, zsh, dash] + - list: shell_mgmt_binaries + items: [add-shell, remove-shell] + - macro: shell_procs + condition: (proc.name in (shell_binaries)) + - list: coreutils_binaries + items: [ + truncate, sha1sum, numfmt, fmt, fold, uniq, cut, who, + groups, csplit, sort, expand, printf, printenv, unlink, tee, chcon, stat, + basename, split, nice, "yes", whoami, sha224sum, hostid, users, stdbuf, + base64, unexpand, cksum, od, paste, nproc, pathchk, sha256sum, wc, test, + comm, arch, du, factor, sha512sum, md5sum, tr, runcon, env, dirname, + tsort, join, shuf, install, logname, pinky, nohup, expr, pr, tty, timeout, + tail, "[", seq, sha384sum, nl, head, id, mkfifo, sum, dircolors, ptx, shred, + tac, link, chroot, vdir, chown, touch, ls, dd, uname, "true", pwd, date, + chgrp, chmod, mktemp, cat, mknod, sync, ln, "false", rm, mv, cp, echo, + readlink, sleep, stty, mkdir, df, dir, rmdir, touch + ] + - list: login_binaries + items: [ + login, systemd, '"(systemd)"', systemd-logind, su, + nologin, faillog, lastlog, newgrp, sg + ] + - list: passwd_binaries + items: [ + shadowconfig, grpck, pwunconv, grpconv, pwck, + groupmod, vipw, pwconv, useradd, newusers, cppw, chpasswd, usermod, + groupadd, groupdel, grpunconv, chgpasswd, userdel, chage, chsh, + gpasswd, chfn, expiry, passwd, vigr, cpgr + ] + - list: shadowutils_binaries + items: [ + chage, gpasswd, lastlog, newgrp, sg, adduser, deluser, chpasswd, + groupadd, groupdel, addgroup, delgroup, groupmems, groupmod, grpck, grpconv, grpunconv, + newusers, pwck, pwconv, pwunconv, useradd, userdel, usermod, vigr, vipw, unix_chkpwd + ] + - list: sysdigcloud_binaries + items: [setup-backend, dragent, sdchecks] + - list: docker_binaries + items: [docker, dockerd, exe, docker-compose, docker-entrypoi, docker-runc-cur, docker-current] + - list: k8s_binaries + items: [hyperkube, skydns, kube2sky, exechealthz] + - list: lxd_binaries + items: [lxd, lxcfs] + - list: http_server_binaries + items: [nginx, httpd, httpd-foregroun, lighttpd, apache, apache2] + - list: db_server_binaries + items: [mysqld, postgres, sqlplus] + - list: mysql_mgmt_binaries + items: [mysql_install_d, mysql_ssl_rsa_s] + - list: postgres_mgmt_binaries + items: [pg_dumpall, pg_ctl, pg_lsclusters, pg_ctlcluster] + - list: db_mgmt_binaries + items: [mysql_mgmt_binaries, postgres_mgmt_binaries] + - list: nosql_server_binaries + items: [couchdb, memcached, redis-server, rabbitmq-server, mongod] + - list: gitlab_binaries + items: [gitlab-shell, gitlab-mon, gitlab-runner-b, git] + - macro: server_procs + condition: proc.name in (http_server_binaries, db_server_binaries, docker_binaries, sshd) + - list: rpm_binaries + items: [dnf, rpm, rpmkey, yum, '"75-system-updat"', rhsmcertd-worke, subscription-ma, + repoquery, rpmkeys, rpmq, yum-cron, yum-config-mana, yum-debug-dump, + abrt-action-sav, rpmdb_stat, microdnf] + - macro: rpm_procs + condition: proc.name in (rpm_binaries) or proc.name in (salt-minion) + - list: deb_binaries + items: [dpkg, dpkg-preconfigu, dpkg-reconfigur, dpkg-divert, apt, apt-get, aptitude, + frontend, preinst, add-apt-reposit, apt-auto-remova, apt-key, + apt-listchanges, unattended-upgr, apt-add-reposit + ] + - list: package_mgmt_binaries + items: [rpm_binaries, deb_binaries, update-alternat, gem, pip, pip3, sane-utils.post, alternatives, chef-client] + - macro: package_mgmt_procs + condition: proc.name in (package_mgmt_binaries) + - macro: coreos_write_ssh_dir + condition: (proc.name=update-ssh-keys and fd.name startswith /home/core/.ssh) + - macro: run_by_package_mgmt_binaries + condition: proc.aname in (package_mgmt_binaries, needrestart) + - list: ssl_mgmt_binaries + items: [ca-certificates] + - list: dhcp_binaries + items: [dhclient, dhclient-script] + - list: userexec_binaries + items: [sudo, su, suexec] + - list: known_setuid_binaries + items: [ + sshd, dbus-daemon-lau, ping, ping6, critical-stack-, pmmcli, + filemng, PassengerAgent, bwrap, osdetect, nginxmng, sw-engine-fpm, + start-stop-daem + ] + - list: user_mgmt_binaries + items: [login_binaries, passwd_binaries, shadowutils_binaries] + - list: dev_creation_binaries + items: [blkid, rename_device, update_engine, sgdisk] + - list: hids_binaries + items: [aide] + - list: vpn_binaries + items: [openvpn] + - list: nomachine_binaries + items: [nxexec, nxnode.bin, nxserver.bin, nxclient.bin] + - macro: system_procs + condition: proc.name in (coreutils_binaries, user_mgmt_binaries) + - list: mail_binaries + items: [ + sendmail, sendmail-msp, postfix, procmail, exim4, + pickup, showq, mailq, dovecot, imap-login, imap, + mailmng-core, pop3-login, dovecot-lda, pop3 + ] + - list: mail_config_binaries + items: [ + update_conf, parse_mc, makemap_hash, newaliases, update_mk, update_tlsm4, + update_db, update_mc, ssmtp.postinst, mailq, postalias, postfix.config., + postfix.config, postfix-script + ] + - list: sensitive_file_names + items: [/etc/shadow, /etc/sudoers, /etc/pam.conf] + - macro: sensitive_files + condition: > + fd.name startswith /etc and + (fd.name in (sensitive_file_names) + or fd.directory in (/etc/sudoers.d, /etc/pam.d)) + - macro: proc_is_new + condition: proc.duration <= 5000000000 + - macro: inbound + condition: > + (((evt.type in (accept,listen) and evt.dir=<)) or + (fd.typechar = 4 or fd.typechar = 6) and + (fd.ip != "0.0.0.0" and fd.net != "127.0.0.0/8") and + (evt.rawres >= 0 or evt.res = EINPROGRESS)) + - macro: outbound + condition: > + (((evt.type = connect and evt.dir=<)) or + (fd.typechar = 4 or fd.typechar = 6) and + (fd.ip != "0.0.0.0" and fd.net != "127.0.0.0/8") and + (evt.rawres >= 0 or evt.res = EINPROGRESS)) + - macro: inbound_outbound + condition: > + (((evt.type in (accept,listen,connect) and evt.dir=<)) or + (fd.typechar = 4 or fd.typechar = 6) and + (fd.ip != "0.0.0.0" and fd.net != "127.0.0.0/8") and + (evt.rawres >= 0 or evt.res = EINPROGRESS)) + - macro: ssh_port + condition: fd.sport=22 + - macro: allowed_ssh_hosts + condition: ssh_port + - rule: Disallowed SSH Connection + desc: Detect any new ssh connection to a host other than those in an allowed group of hosts + condition: (inbound_outbound) and ssh_port and not allowed_ssh_hosts + output: Disallowed SSH Connection (command=%proc.cmdline connection=%fd.name user=%user.name) + priority: NOTICE + tags: [network] + - macro: container + condition: container.id != host + - macro: interactive + condition: > + ((proc.aname=sshd and proc.name != sshd) or + proc.name=systemd-logind or proc.name=login) + - list: cron_binaries + items: [anacron, cron, crond, crontab] + - list: needrestart_binaries + items: [needrestart, 10-dpkg, 20-rpm, 30-pacman] + - list: sshkit_script_binaries + items: [10_etc_sudoers., 10_passwd_group] + - list: plesk_binaries + items: [sw-engine, sw-engine-fpm, sw-engine-kv, filemng, f2bmng] + - macro: system_users + condition: user.name in (bin, daemon, games, lp, mail, nobody, sshd, sync, uucp, www-data) + - macro: parent_ansible_running_python + condition: (proc.pname in (python, pypy) and proc.pcmdline contains ansible) + - macro: parent_bro_running_python + condition: (proc.pname=python and proc.cmdline contains /usr/share/broctl) + - macro: parent_python_running_denyhosts + condition: > + (proc.cmdline startswith "denyhosts.py /usr/bin/denyhosts.py" or + (proc.pname=python and + (proc.pcmdline contains /usr/sbin/denyhosts or + proc.pcmdline contains /usr/local/bin/denyhosts.py))) + - macro: parent_python_running_sdchecks + condition: > + (proc.pname in (python, python2.7) and + (proc.pcmdline contains /opt/draios/bin/sdchecks)) + - macro: parent_linux_image_upgrade_script + condition: proc.pname startswith linux-image- + - macro: parent_java_running_echo + condition: (proc.pname=java and proc.cmdline startswith "sh -c echo") + - macro: parent_scripting_running_builds + condition: > + (proc.pname in (php,php5-fpm,php-fpm7.1,python,ruby,ruby2.3,ruby2.1,node,conda) and ( + proc.cmdline startswith "sh -c git" or + proc.cmdline startswith "sh -c date" or + proc.cmdline startswith "sh -c /usr/bin/g++" or + proc.cmdline startswith "sh -c /usr/bin/gcc" or + proc.cmdline startswith "sh -c gcc" or + proc.cmdline startswith "sh -c if type gcc" or + proc.cmdline startswith "sh -c cd '/var/www/edi/';LC_ALL=en_US.UTF-8 git" or + proc.cmdline startswith "sh -c /var/www/edi/bin/sftp.sh" or + proc.cmdline startswith "sh -c /usr/src/app/crxlsx/bin/linux/crxlsx" or + proc.cmdline startswith "sh -c make parent" or + proc.cmdline startswith "node /jenkins/tools" or + proc.cmdline startswith "sh -c '/usr/bin/node'" or + proc.cmdline startswith "sh -c stty -a |" or + proc.pcmdline startswith "node /opt/nodejs/bin/yarn" or + proc.pcmdline startswith "node /usr/local/bin/yarn" or + proc.pcmdline startswith "node /root/.config/yarn" or + proc.pcmdline startswith "node /opt/yarn/bin/yarn.js")) + - macro: httpd_writing_ssl_conf + condition: > + (proc.pname=run-httpd and + (proc.cmdline startswith "sed -ri" or proc.cmdline startswith "sed -i") and + (fd.name startswith /etc/httpd/conf.d/ or fd.name startswith /etc/httpd/conf)) + - macro: userhelper_writing_etc_security + condition: (proc.name=userhelper and fd.name startswith /etc/security) + - macro: parent_Xvfb_running_xkbcomp + condition: (proc.pname=Xvfb and proc.cmdline startswith 'sh -c "/usr/bin/xkbcomp"') + - macro: parent_nginx_running_serf + condition: (proc.pname=nginx and proc.cmdline startswith "sh -c serf") + - macro: parent_node_running_npm + condition: (proc.pcmdline startswith "node /usr/local/bin/npm" or + proc.pcmdline startswith "node /usr/local/nodejs/bin/npm" or + proc.pcmdline startswith "node /opt/rh/rh-nodejs6/root/usr/bin/npm") + - macro: parent_java_running_sbt + condition: (proc.pname=java and proc.pcmdline contains sbt-launch.jar) + - list: known_container_shell_spawn_cmdlines + items: [] + - list: known_shell_spawn_binaries + items: [] + - macro: ansible_running_python + condition: (proc.name in (python, pypy) and proc.cmdline contains ansible) + - macro: python_running_chef + condition: (proc.name=python and (proc.cmdline contains yum-dump.py or proc.cmdline="python /usr/bin/chef-monitor.py")) + - macro: python_running_denyhosts + condition: > + (proc.name=python and + (proc.cmdline contains /usr/sbin/denyhosts or + proc.cmdline contains /usr/local/bin/denyhosts.py)) + - macro: run_by_qualys + condition: > + (proc.pname=qualys-cloud-ag or + proc.aname[2]=qualys-cloud-ag or + proc.aname[3]=qualys-cloud-ag or + proc.aname[4]=qualys-cloud-ag) + - macro: run_by_sumologic_securefiles + condition: > + ((proc.cmdline="usermod -a -G sumologic_collector" or + proc.cmdline="groupadd sumologic_collector") and + (proc.pname=secureFiles.sh and proc.aname[2]=java)) + - macro: run_by_yum + condition: ((proc.pname=sh and proc.aname[2]=yum) or + (proc.aname[2]=sh and proc.aname[3]=yum)) + - macro: run_by_ms_oms + condition: > + (proc.aname[3] startswith omsagent- or + proc.aname[3] startswith scx-) + - macro: run_by_google_accounts_daemon + condition: > + (proc.aname[1] startswith google_accounts or + proc.aname[2] startswith google_accounts) + - macro: run_by_chef + condition: (proc.aname[2]=chef_command_wr or proc.aname[3]=chef_command_wr or + proc.aname[2]=chef-client or proc.aname[3]=chef-client or + proc.name=chef-client) + - macro: run_by_adclient + condition: (proc.aname[2]=adclient or proc.aname[3]=adclient or proc.aname[4]=adclient) + - macro: run_by_centrify + condition: (proc.aname[2]=centrify or proc.aname[3]=centrify or proc.aname[4]=centrify) + - macro: run_by_puppet + condition: (proc.aname[2]=puppet or proc.aname[3]=puppet) + - macro: run_by_foreman + condition: > + (user.name=foreman and + (proc.pname in (rake, ruby, scl) and proc.aname[5] in (tfm-rake,tfm-ruby)) or + (proc.pname=scl and proc.aname[2] in (tfm-rake,tfm-ruby))) + - macro: java_running_sdjagent + condition: proc.name=java and proc.cmdline contains sdjagent.jar + - macro: kubelet_running_loopback + condition: (proc.pname=kubelet and proc.name=loopback) + - macro: python_mesos_marathon_scripting + condition: (proc.pcmdline startswith "python3 /marathon-lb/marathon_lb.py") + - macro: splunk_running_forwarder + condition: (proc.pname=splunkd and proc.cmdline startswith "sh -c /opt/splunkforwarder") + - macro: parent_supervise_running_multilog + condition: (proc.name=multilog and proc.pname=supervise) + - macro: supervise_writing_status + condition: (proc.name in (supervise,svc) and fd.name startswith "/etc/sb/") + - macro: pki_realm_writing_realms + condition: (proc.cmdline startswith "bash /usr/local/lib/pki/pki-realm" and fd.name startswith /etc/pki/realms) + - macro: htpasswd_writing_passwd + condition: (proc.name=htpasswd and fd.name=/etc/nginx/.htpasswd) + - macro: lvprogs_writing_conf + condition: > + (proc.name in (dmeventd,lvcreate,pvscan) and + (fd.name startswith /etc/lvm/archive or + fd.name startswith /etc/lvm/backup or + fd.name startswith /etc/lvm/cache)) + - macro: ovsdb_writing_openvswitch + condition: (proc.name=ovsdb-server and fd.directory=/etc/openvswitch) + - macro: perl_running_plesk + condition: (proc.cmdline startswith "perl /opt/psa/admin/bin/plesk_agent_manager" or + proc.pcmdline startswith "perl /opt/psa/admin/bin/plesk_agent_manager") + - macro: perl_running_updmap + condition: (proc.cmdline startswith "perl /usr/bin/updmap") + - macro: perl_running_centrifydc + condition: (proc.cmdline startswith "perl /usr/share/centrifydc") + - macro: parent_ucf_writing_conf + condition: (proc.pname=ucf and proc.aname[2]=frontend) + - macro: consul_template_writing_conf + condition: > + ((proc.name=consul-template and fd.name startswith /etc/haproxy) or + (proc.name=reload.sh and proc.aname[2]=consul-template and fd.name startswith /etc/ssl)) + - macro: countly_writing_nginx_conf + condition: (proc.cmdline startswith "nodejs /opt/countly/bin" and fd.name startswith /etc/nginx) + - list: ms_oms_binaries + items: [omi.postinst, omsconfig.posti, scx.postinst, omsadmin.sh, omiagent] + - macro: ms_oms_writing_conf + condition: > + ((proc.name in (omiagent,omsagent,in_heartbeat_r*,omsadmin.sh,PerformInventor) + or proc.pname in (ms_oms_binaries) + or proc.aname[2] in (ms_oms_binaries)) + and (fd.name startswith /etc/opt/omi or fd.name startswith /etc/opt/microsoft/omsagent)) + - macro: ms_scx_writing_conf + condition: (proc.name in (GetLinuxOS.sh) and fd.name startswith /etc/opt/microsoft/scx) + - macro: azure_scripts_writing_conf + condition: (proc.pname startswith "bash /var/lib/waagent/" and fd.name startswith /etc/azure) + - macro: azure_networkwatcher_writing_conf + condition: (proc.name in (NetworkWatcherA) and fd.name=/etc/init.d/AzureNetworkWatcherAgent) + - macro: couchdb_writing_conf + condition: (proc.name=beam.smp and proc.cmdline contains couchdb and fd.name startswith /etc/couchdb) + - macro: update_texmf_writing_conf + condition: (proc.name=update-texmf and fd.name startswith /etc/texmf) + - macro: slapadd_writing_conf + condition: (proc.name=slapadd and fd.name startswith /etc/ldap) + - macro: openldap_writing_conf + condition: (proc.pname=run-openldap.sh and fd.name startswith /etc/openldap) + - macro: ucpagent_writing_conf + condition: (proc.name=apiserver and container.image startswith docker/ucp-agent and fd.name=/etc/authorization_config.cfg) + - macro: iscsi_writing_conf + condition: (proc.name=iscsiadm and fd.name startswith /etc/iscsi) + - macro: symantec_writing_conf + condition: > + ((proc.name=symcfgd and fd.name startswith /etc/symantec) or + (proc.name=navdefutil and fd.name=/etc/symc-defutils.conf)) + - macro: liveupdate_writing_conf + condition: (proc.cmdline startswith "java LiveUpdate" and fd.name in (/etc/liveupdate.conf, /etc/Product.Catalog.JavaLiveUpdate)) + - macro: sosreport_writing_files + condition: > + (proc.name=urlgrabber-ext- and proc.aname[3]=sosreport and + (fd.name startswith /etc/pkt/nssdb or fd.name startswith /etc/pki/nssdb)) + - macro: pkgmgmt_progs_writing_pki + condition: > + (proc.name=urlgrabber-ext- and proc.pname in (yum, yum-cron, repoquery) and + (fd.name startswith /etc/pkt/nssdb or fd.name startswith /etc/pki/nssdb)) + - macro: update_ca_trust_writing_pki + condition: (proc.pname=update-ca-trust and proc.name=trust and fd.name startswith /etc/pki) + - macro: brandbot_writing_os_release + condition: proc.name=brandbot and fd.name=/etc/os-release + - macro: selinux_writing_conf + condition: (proc.name in (semodule,genhomedircon,sefcontext_comp) and fd.name startswith /etc/selinux) + - list: veritas_binaries + items: [vxconfigd, sfcache, vxclustadm, vxdctl, vxprint, vxdmpadm, vxdisk, vxdg, vxassist, vxtune] + - macro: veritas_driver_script + condition: (proc.cmdline startswith "perl /opt/VRTSsfmh/bin/mh_driver.pl") + - macro: veritas_progs + condition: (proc.name in (veritas_binaries) or veritas_driver_script) + - macro: veritas_writing_config + condition: (veritas_progs and (fd.name startswith /etc/vx or fd.name startswith /etc/opt/VRTS or fd.name startswith /etc/vom)) + - macro: nginx_writing_conf + condition: (proc.name=nginx and fd.name startswith /etc/nginx) + - macro: nginx_writing_certs + condition: > + (((proc.name=openssl and proc.pname=nginx-launch.sh) or proc.name=nginx-launch.sh) and fd.name startswith /etc/nginx/certs) + - macro: chef_client_writing_conf + condition: (proc.pcmdline startswith "chef-client /opt/gitlab" and fd.name startswith /etc/gitlab) + - macro: centrify_writing_krb + condition: (proc.name in (adjoin,addns) and fd.name startswith /etc/krb5) + - macro: cockpit_writing_conf + condition: > + ((proc.pname=cockpit-kube-la or proc.aname[2]=cockpit-kube-la) + and fd.name startswith /etc/cockpit) + - macro: ipsec_writing_conf + condition: (proc.name=start-ipsec.sh and fd.directory=/etc/ipsec) + - macro: exe_running_docker_save + condition: (proc.cmdline startswith "exe /var/lib/docker" and proc.pname in (dockerd, docker)) + - macro: sed_temporary_file + condition: (proc.name=sed and fd.name startswith "/etc/sed") + - macro: python_running_get_pip + condition: (proc.cmdline startswith "python get-pip.py") + - macro: python_running_ms_oms + condition: (proc.cmdline startswith "python /var/lib/waagent/") + - macro: gugent_writing_guestagent_log + condition: (proc.name=gugent and fd.name=GuestAgent.log) + - macro: dse_writing_tmp + condition: (proc.name=dse-entrypoint and fd.name=/root/tmp__) + - macro: zap_writing_state + condition: (proc.name=java and proc.cmdline contains "jar /zap" and fd.name startswith /root/.ZAP) + - macro: airflow_writing_state + condition: (proc.name=airflow and fd.name startswith /root/airflow) + - macro: rpm_writing_root_rpmdb + condition: (proc.name=rpm and fd.directory=/root/.rpmdb) + - macro: maven_writing_groovy + condition: (proc.name=java and proc.cmdline contains "classpath /usr/local/apache-maven" and fd.name startswith /root/.groovy) + - rule: Write below binary dir + desc: an attempt to write to any file below a set of binary directories + condition: > + bin_dir and evt.dir = < and open_write + and not package_mgmt_procs + and not exe_running_docker_save + and not python_running_get_pip + and not python_running_ms_oms + output: > + File below a known binary directory opened for writing (user=%user.name + command=%proc.cmdline file=%fd.name parent=%proc.pname pcmdline=%proc.pcmdline gparent=%proc.aname[2]) + priority: ERROR + tags: [filesystem] + - list: monitored_directories + items: [/boot, /lib, /lib64, /usr/lib, /usr/local/lib, /usr/local/sbin, /usr/local/bin, /root/.ssh, /etc/cardserver] + - macro: user_ssh_directory + condition: (fd.name startswith '/home' and fd.name contains '.ssh') + - macro: mkinitramfs_writing_boot + condition: (proc.pname in (mkinitramfs, update-initramf) and fd.directory=/boot) + - macro: monitored_dir + condition: > + (fd.directory in (monitored_directories) + or user_ssh_directory) + and not mkinitramfs_writing_boot + - rule: Write below monitored dir + desc: an attempt to write to any file below a set of binary directories + condition: > + evt.dir = < and open_write and monitored_dir + and not package_mgmt_procs + and not coreos_write_ssh_dir + and not exe_running_docker_save + and not python_running_get_pip + and not python_running_ms_oms + output: > + File below a monitored directory opened for writing (user=%user.name + command=%proc.cmdline file=%fd.name parent=%proc.pname pcmdline=%proc.pcmdline gparent=%proc.aname[2]) + priority: ERROR + tags: [filesystem] + - list: safe_etc_dirs + items: [/etc/cassandra, /etc/ssl/certs/java, /etc/logstash, /etc/nginx/conf.d, /etc/container_environment, /etc/hrmconfig] + - macro: fluentd_writing_conf_files + condition: (proc.name=start-fluentd and fd.name in (/etc/fluent/fluent.conf, /etc/td-agent/td-agent.conf)) + - macro: qualys_writing_conf_files + condition: (proc.name=qualys-cloud-ag and fd.name=/etc/qualys/cloud-agent/qagent-log.conf) + - macro: git_writing_nssdb + condition: (proc.name=git-remote-http and fd.directory=/etc/pki/nssdb) + - macro: plesk_writing_keys + condition: (proc.name in (plesk_binaries) and fd.name startswith /etc/sw/keys) + - macro: plesk_install_writing_apache_conf + condition: (proc.cmdline startswith "bash -hB /usr/lib/plesk-9.0/services/webserver.apache configure" + and fd.name="/etc/apache2/apache2.conf.tmp") + - macro: plesk_running_mktemp + condition: (proc.name=mktemp and proc.aname[3] in (plesk_binaries)) + - macro: networkmanager_writing_resolv_conf + condition: proc.aname[2]=nm-dispatcher and fd.name=/etc/resolv.conf + - macro: add_shell_writing_shells_tmp + condition: (proc.name=add-shell and fd.name=/etc/shells.tmp) + - macro: duply_writing_exclude_files + condition: (proc.name=touch and proc.pcmdline startswith "bash /usr/bin/duply" and fd.name startswith "/etc/duply") + - macro: xmlcatalog_writing_files + condition: (proc.name=update-xmlcatal and fd.directory=/etc/xml) + - macro: datadog_writing_conf + condition: ((proc.cmdline startswith "python /opt/datadog-agent" or + proc.cmdline startswith "entrypoint.sh /entrypoint.sh datadog start" or + proc.cmdline startswith "agent.py /opt/datadog-agent") + and fd.name startswith "/etc/dd-agent") + - macro: curl_writing_pki_db + condition: (proc.name=curl and fd.directory=/etc/pki/nssdb) + - macro: haproxy_writing_conf + condition: ((proc.name in (update-haproxy-,haproxy_reload.) or proc.pname in (update-haproxy-,haproxy_reload,haproxy_reload.)) + and (fd.name=/etc/openvpn/client.map or fd.name startswith /etc/haproxy)) + - macro: java_writing_conf + condition: (proc.name=java and fd.name=/etc/.java/.systemPrefs/.system.lock) + - macro: rabbitmq_writing_conf + condition: (proc.name=rabbitmq-server and fd.directory=/etc/rabbitmq) + - macro: rook_writing_conf + condition: (proc.name=toolbox.sh and container.image startswith rook/toolbox + and fd.directory=/etc/ceph) + - macro: httpd_writing_conf_logs + condition: (proc.name=httpd and fd.name startswith /etc/httpd/) + - macro: mysql_writing_conf + condition: > + ((proc.name in (start-mysql.sh, run-mysqld) or proc.pname=start-mysql.sh) and + (fd.name startswith /etc/mysql or fd.directory=/etc/my.cnf.d)) + - macro: redis_writing_conf + condition: > + (proc.name in (run-redis, redis-launcher.) and fd.name=/etc/redis.conf or fd.name startswith /etc/redis) + - macro: openvpn_writing_conf + condition: (proc.name in (openvpn,openvpn-entrypo) and fd.name startswith /etc/openvpn) + - macro: php_handlers_writing_conf + condition: (proc.name=php_handlers_co and fd.name=/etc/psa/php_versions.json) + - macro: sed_writing_temp_file + condition: > + ((proc.aname[3]=cron_start.sh and fd.name startswith /etc/security/sed) or + (proc.name=sed and (fd.name startswith /etc/apt/sources.list.d/sed or + fd.name startswith /etc/apt/sed or + fd.name startswith /etc/apt/apt.conf.d/sed))) + - macro: cron_start_writing_pam_env + condition: (proc.cmdline="bash /usr/sbin/start-cron" and fd.name=/etc/security/pam_env.conf) + - macro: dpkg_scripting + condition: (proc.aname[2] in (dpkg-reconfigur, dpkg-preconfigu)) + - macro: user_known_write_etc_conditions + condition: proc.name=confd + - macro: write_etc_common + condition: > + etc_dir and evt.dir = < and open_write + and proc_name_exists + and not proc.name in (passwd_binaries, shadowutils_binaries, sysdigcloud_binaries, + package_mgmt_binaries, ssl_mgmt_binaries, dhcp_binaries, + dev_creation_binaries, shell_mgmt_binaries, + mail_config_binaries, + sshkit_script_binaries, + ldconfig.real, ldconfig, confd, gpg, insserv, + apparmor_parser, update-mime, tzdata.config, tzdata.postinst, + systemd, systemd-machine, systemd-sysuser, + debconf-show, rollerd, bind9.postinst, sv, + gen_resolvconf., update-ca-certi, certbot, runsv, + qualys-cloud-ag, locales.postins, nomachine_binaries, + adclient, certutil, crlutil, pam-auth-update, parallels_insta, + openshift-launc, update-rc.d) + and not proc.pname in (sysdigcloud_binaries, mail_config_binaries, hddtemp.postins, sshkit_script_binaries, locales.postins, deb_binaries, dhcp_binaries) + and not fd.name pmatch (safe_etc_dirs) + and not fd.name in (/etc/container_environment.sh, /etc/container_environment.json, /etc/motd, /etc/motd.svc) + and not sed_temporary_file + and not exe_running_docker_save + and not ansible_running_python + and not python_running_denyhosts + and not fluentd_writing_conf_files + and not user_known_write_etc_conditions + and not run_by_centrify + and not run_by_adclient + and not qualys_writing_conf_files + and not git_writing_nssdb + and not plesk_writing_keys + and not plesk_install_writing_apache_conf + and not plesk_running_mktemp + and not networkmanager_writing_resolv_conf + and not run_by_chef + and not add_shell_writing_shells_tmp + and not duply_writing_exclude_files + and not xmlcatalog_writing_files + and not parent_supervise_running_multilog + and not supervise_writing_status + and not pki_realm_writing_realms + and not htpasswd_writing_passwd + and not lvprogs_writing_conf + and not ovsdb_writing_openvswitch + and not datadog_writing_conf + and not curl_writing_pki_db + and not haproxy_writing_conf + and not java_writing_conf + and not dpkg_scripting + and not parent_ucf_writing_conf + and not rabbitmq_writing_conf + and not rook_writing_conf + and not php_handlers_writing_conf + and not sed_writing_temp_file + and not cron_start_writing_pam_env + and not httpd_writing_conf_logs + and not mysql_writing_conf + and not openvpn_writing_conf + and not consul_template_writing_conf + and not countly_writing_nginx_conf + and not ms_oms_writing_conf + and not ms_scx_writing_conf + and not azure_scripts_writing_conf + and not azure_networkwatcher_writing_conf + and not couchdb_writing_conf + and not update_texmf_writing_conf + and not slapadd_writing_conf + and not symantec_writing_conf + and not liveupdate_writing_conf + and not sosreport_writing_files + and not selinux_writing_conf + and not veritas_writing_config + and not nginx_writing_conf + and not nginx_writing_certs + and not chef_client_writing_conf + and not centrify_writing_krb + and not cockpit_writing_conf + and not ipsec_writing_conf + and not httpd_writing_ssl_conf + and not userhelper_writing_etc_security + and not pkgmgmt_progs_writing_pki + and not update_ca_trust_writing_pki + and not brandbot_writing_os_release + and not redis_writing_conf + and not openldap_writing_conf + and not ucpagent_writing_conf + and not iscsi_writing_conf + - rule: Write below etc + desc: an attempt to write to any file below /etc + condition: write_etc_common + output: "File below /etc opened for writing (user=%user.name command=%proc.cmdline parent=%proc.pname pcmdline=%proc.pcmdline file=%fd.name program=%proc.name gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4])" + priority: ERROR + tags: [filesystem] + - list: known_root_files + items: [/root/.monit.state, /root/.auth_tokens, /root/.bash_history, /root/.ash_history, /root/.aws/credentials, + /root/.viminfo.tmp, /root/.lesshst, /root/.bzr.log, /root/.gitconfig.lock, /root/.babel.json, /root/.localstack, + /root/.node_repl_history, /root/.mongorc.js, /root/.dbshell, /root/.augeas/history, /root/.rnd, /root/.wget-hsts] + - list: known_root_directories + items: [/root/.oracle_jre_usage, /root/.ssh, /root/.subversion, /root/.nami] + - macro: known_root_conditions + condition: (fd.name startswith /root/orcexec. + or fd.name startswith /root/.m2 + or fd.name startswith /root/.npm + or fd.name startswith /root/.pki + or fd.name startswith /root/.ivy2 + or fd.name startswith /root/.config/Cypress + or fd.name startswith /root/.config/pulse + or fd.name startswith /root/.config/configstore + or fd.name startswith /root/jenkins/workspace + or fd.name startswith /root/.jenkins + or fd.name startswith /root/.cache + or fd.name startswith /root/.sbt + or fd.name startswith /root/.java + or fd.name startswith /root/.glide + or fd.name startswith /root/.sonar + or fd.name startswith /root/.v8flag + or fd.name startswith /root/infaagent + or fd.name startswith /root/.local/lib/python + or fd.name startswith /root/.pm2 + or fd.name startswith /root/.gnupg + or fd.name startswith /root/.pgpass + or fd.name startswith /root/.theano + or fd.name startswith /root/.gradle + or fd.name startswith /root/.android + or fd.name startswith /root/.ansible + or fd.name startswith /root/.crashlytics + or fd.name startswith /root/.dbus + or fd.name startswith /root/.composer + or fd.name startswith /root/.gconf + or fd.name startswith /root/.nv + or fd.name startswith /root/.local/share/jupyter + or fd.name startswith /root/oradiag_root + or fd.name startswith /root/workspace + or fd.name startswith /root/jvm + or fd.name startswith /root/.node-gyp) + - rule: Write below root + desc: an attempt to write to any file directly below / or /root + condition: > + root_dir and evt.dir = < and open_write + and not fd.name in (known_root_files) + and not fd.directory in (known_root_directories) + and not exe_running_docker_save + and not gugent_writing_guestagent_log + and not dse_writing_tmp + and not zap_writing_state + and not airflow_writing_state + and not rpm_writing_root_rpmdb + and not maven_writing_groovy + and not known_root_conditions + output: "File below / or /root opened for writing (user=%user.name command=%proc.cmdline parent=%proc.pname file=%fd.name program=%proc.name)" + priority: ERROR + tags: [filesystem] + - macro: cmp_cp_by_passwd + condition: proc.name in (cmp, cp) and proc.pname in (passwd, run-parts) + - rule: Read sensitive file trusted after startup + desc: > + an attempt to read any sensitive file (e.g. files containing user/password/authentication + information) by a trusted program after startup. Trusted programs might read these files + at startup to load initial state, but not afterwards. + condition: sensitive_files and open_read and server_procs and not proc_is_new and proc.name!="sshd" + output: > + Sensitive file opened for reading by trusted program after startup (user=%user.name + command=%proc.cmdline parent=%proc.pname file=%fd.name parent=%proc.pname gparent=%proc.aname[2]) + priority: WARNING + tags: [filesystem] + - list: read_sensitive_file_binaries + items: [ + iptables, ps, lsb_release, check-new-relea, dumpe2fs, accounts-daemon, sshd, + vsftpd, systemd, mysql_install_d, psql, screen, debconf-show, sa-update, + pam-auth-update, pam-config, /usr/sbin/spamd, polkit-agent-he, lsattr, file, sosreport, + scxcimservera, adclient, rtvscand, cockpit-session, userhelper, ossec-syscheckd + ] + - macro: user_read_sensitive_file_conditions + condition: cmp_cp_by_passwd + - rule: Read sensitive file untrusted + desc: > + an attempt to read any sensitive file (e.g. files containing user/password/authentication + information). Exceptions are made for known trusted programs. + condition: > + sensitive_files and open_read + and proc_name_exists + and not proc.name in (user_mgmt_binaries, userexec_binaries, package_mgmt_binaries, + cron_binaries, read_sensitive_file_binaries, shell_binaries, hids_binaries, + vpn_binaries, mail_config_binaries, nomachine_binaries, sshkit_script_binaries, + in.proftpd, mandb, salt-minion, postgres_mgmt_binaries) + and not cmp_cp_by_passwd + and not ansible_running_python + and not proc.cmdline contains /usr/bin/mandb + and not run_by_qualys + and not run_by_chef + and not user_read_sensitive_file_conditions + and not perl_running_plesk + and not perl_running_updmap + and not veritas_driver_script + and not perl_running_centrifydc + output: > + Sensitive file opened for reading by non-trusted program (user=%user.name program=%proc.name + command=%proc.cmdline file=%fd.name parent=%proc.pname gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]) + priority: WARNING + tags: [filesystem] + - rule: Write below rpm database + desc: an attempt to write to the rpm database by any non-rpm related program + condition: fd.name startswith /var/lib/rpm and open_write and not rpm_procs and not ansible_running_python and not python_running_chef + output: "Rpm database opened for writing by a non-rpm program (command=%proc.cmdline file=%fd.name parent=%proc.pname pcmdline=%proc.pcmdline)" + priority: ERROR + tags: [filesystem, software_mgmt] + - macro: postgres_running_wal_e + condition: (proc.pname=postgres and proc.cmdline startswith "sh -c envdir /etc/wal-e.d/env /usr/local/bin/wal-e") + - macro: redis_running_prepost_scripts + condition: (proc.aname[2]=redis-server and (proc.cmdline contains "redis-server.post-up.d" or proc.cmdline contains "redis-server.pre-up.d")) + - macro: rabbitmq_running_scripts + condition: > + (proc.pname=beam.smp and + (proc.cmdline startswith "sh -c exec ps" or + proc.cmdline startswith "sh -c exec inet_gethost" or + proc.cmdline= "sh -s unix:cmd" or + proc.cmdline= "sh -c exec /bin/sh -s unix:cmd 2>&1")) + - macro: rabbitmqctl_running_scripts + condition: (proc.aname[2]=rabbitmqctl and proc.cmdline startswith "sh -c ") + - macro: run_by_appdynamics + condition: (proc.pname=java and proc.pcmdline startswith "java -jar -Dappdynamics") + - rule: DB program spawned process + desc: > + a database-server related program spawned a new process other than itself. + This shouldn\'t occur and is a follow on from some SQL injection attacks. + condition: > + proc.pname in (db_server_binaries) + and spawned_process + and not proc.name in (db_server_binaries) + and not postgres_running_wal_e + output: > + Database-related program spawned process other than itself (user=%user.name + program=%proc.cmdline parent=%proc.pname) + priority: NOTICE + tags: [process, database] + - rule: Modify binary dirs + desc: an attempt to modify any file below a set of binary directories. + condition: (bin_dir_rename) and modify and not package_mgmt_procs and not exe_running_docker_save + output: > + File below known binary directory renamed/removed (user=%user.name command=%proc.cmdline + pcmdline=%proc.pcmdline operation=%evt.type file=%fd.name %evt.args) + priority: ERROR + tags: [filesystem] + - rule: Mkdir binary dirs + desc: an attempt to create a directory below a set of binary directories. + condition: mkdir and bin_dir_mkdir and not package_mgmt_procs + output: > + Directory below known binary directory created (user=%user.name + command=%proc.cmdline directory=%evt.arg.path) + priority: ERROR + tags: [filesystem] + - list: user_known_change_thread_namespace_binaries + items: [] + - rule: Change thread namespace + desc: > + an attempt to change a program/thread\'s namespace (commonly done + as a part of creating a container) by calling setns. + condition: > + evt.type = setns + and not proc.name in (docker_binaries, k8s_binaries, lxd_binaries, sysdigcloud_binaries, sysdig, nsenter) + and not proc.name in (user_known_change_thread_namespace_binaries) + and not proc.name startswith "runc:" + and not proc.pname in (sysdigcloud_binaries) + and not java_running_sdjagent + and not kubelet_running_loopback + output: > + Namespace change (setns) by unexpected program (user=%user.name command=%proc.cmdline + parent=%proc.pname %container.info) + priority: NOTICE + tags: [process] + - list: protected_shell_spawning_binaries + items: [ + http_server_binaries, db_server_binaries, nosql_server_binaries, mail_binaries, + fluentd, flanneld, splunkd, consul, smbd, runsv, PM2 + ] + - macro: parent_java_running_elasticsearch + condition: (proc.pname=java and proc.pcmdline contains org.elasticsearch.bootstrap.Elasticsearch) + - macro: parent_java_running_activemq + condition: (proc.pname=java and proc.pcmdline contains activemq.jar) + - macro: parent_java_running_cassandra + condition: (proc.pname=java and (proc.pcmdline contains "-Dcassandra.config.loader" or proc.pcmdline contains org.apache.cassandra.service.CassandraDaemon)) + - macro: parent_java_running_jboss_wildfly + condition: (proc.pname=java and proc.pcmdline contains org.jboss) + - macro: parent_java_running_glassfish + condition: (proc.pname=java and proc.pcmdline contains com.sun.enterprise.glassfish) + - macro: parent_java_running_hadoop + condition: (proc.pname=java and proc.pcmdline contains org.apache.hadoop) + - macro: parent_java_running_datastax + condition: (proc.pname=java and proc.pcmdline contains com.datastax) + - macro: nginx_starting_nginx + condition: (proc.pname=nginx and proc.cmdline contains "/usr/sbin/nginx -c /etc/nginx/nginx.conf") + - macro: nginx_running_aws_s3_cp + condition: (proc.pname=nginx and proc.cmdline startswith "sh -c /usr/local/bin/aws s3 cp") + - macro: consul_running_net_scripts + condition: (proc.pname=consul and (proc.cmdline startswith "sh -c curl" or proc.cmdline startswith "sh -c nc")) + - macro: consul_running_alert_checks + condition: (proc.pname=consul and proc.cmdline startswith "sh -c /bin/consul-alerts") + - macro: serf_script + condition: (proc.cmdline startswith "sh -c serf") + - macro: check_process_status + condition: (proc.cmdline startswith "sh -c kill -0 ") + - macro: possibly_node_in_container + condition: (never_true and (proc.pname=node and proc.aname[3]=docker-containe)) + - macro: possibly_parent_java_running_tomcat + condition: (never_true and proc.pname=java and proc.pcmdline contains org.apache.catalina.startup.Bootstrap) + - macro: protected_shell_spawner + condition: > + (proc.aname in (protected_shell_spawning_binaries) + or parent_java_running_elasticsearch + or parent_java_running_activemq + or parent_java_running_cassandra + or parent_java_running_jboss_wildfly + or parent_java_running_glassfish + or parent_java_running_hadoop + or parent_java_running_datastax + or possibly_parent_java_running_tomcat + or possibly_node_in_container) + - list: mesos_shell_binaries + items: [mesos-docker-ex, mesos-slave, mesos-health-ch] + - rule: Run shell untrusted + desc: an attempt to spawn a shell below a non-shell application. Specific applications are monitored. + condition: > + spawned_process + and shell_procs + and proc.pname exists + and protected_shell_spawner + and not proc.pname in (shell_binaries, gitlab_binaries, cron_binaries, user_known_shell_spawn_binaries, + needrestart_binaries, + mesos_shell_binaries, + erl_child_setup, exechealthz, + PM2, PassengerWatchd, c_rehash, svlogd, logrotate, hhvm, serf, + lb-controller, nvidia-installe, runsv, statsite, erlexec) + and not proc.cmdline in (known_shell_spawn_cmdlines) + and not proc.aname in (unicorn_launche) + and not consul_running_net_scripts + and not consul_running_alert_checks + and not nginx_starting_nginx + and not nginx_running_aws_s3_cp + and not run_by_package_mgmt_binaries + and not serf_script + and not check_process_status + and not run_by_foreman + and not python_mesos_marathon_scripting + and not splunk_running_forwarder + and not postgres_running_wal_e + and not redis_running_prepost_scripts + and not rabbitmq_running_scripts + and not rabbitmqctl_running_scripts + and not run_by_appdynamics + and not user_shell_container_exclusions + output: > + Shell spawned by untrusted binary (user=%user.name shell=%proc.name parent=%proc.pname + cmdline=%proc.cmdline pcmdline=%proc.pcmdline gparent=%proc.aname[2] ggparent=%proc.aname[3] + aname[4]=%proc.aname[4] aname[5]=%proc.aname[5] aname[6]=%proc.aname[6] aname[7]=%proc.aname[7]) + priority: DEBUG + tags: [shell] + - macro: trusted_containers + condition: (container.image startswith sysdig/agent or + (container.image startswith sysdig/falco and + not container.image startswith sysdig/falco-event-generator) or + container.image startswith quay.io/sysdig or + container.image startswith sysdig/sysdig or + container.image startswith registry.k8s.io/hyperkube or + container.image startswith quay.io/coreos/flannel or + container.image startswith registry.k8s.io/kube-proxy or + container.image startswith calico/node or + container.image startswith rook/toolbox or + container.image startswith registry.access.redhat.com/openshift3/logging-fluentd or + container.image startswith registry.access.redhat.com/openshift3/logging-elasticsearch or + container.image startswith registry.access.redhat.com/openshift3/metrics-cassandra or + container.image startswith openshift3/ose-sti-builder or + container.image startswith registry.access.redhat.com/openshift3/ose-sti-builder or + container.image startswith registry.access.redhat.com/openshift3/ose-docker-builder or + container.image startswith registry.access.redhat.com/openshift3/image-inspector or + container.image startswith cloudnativelabs/kube-router or + container.image startswith "consul:" or + container.image startswith mesosphere/mesos-slave or + container.image startswith istio/proxy_ or + container.image startswith datadog/docker-dd-agent or + container.image startswith datadog/agent or + container.image startswith docker/ucp-agent or + container.image startswith gliderlabs/logspout) + - macro: user_trusted_containers + condition: (container.image startswith sysdig/agent) + - macro: user_sensitive_mount_containers + condition: (container.image startswith sysdig/agent) + - rule: Launch Privileged Container + desc: Detect the initial process started in a privileged container. Exceptions are made for known trusted images. + condition: > + evt.type=execve and proc.vpid=1 and container + and container.privileged=true + and not trusted_containers + and not user_trusted_containers + output: Privileged container started (user=%user.name command=%proc.cmdline %container.info image=%container.image) + priority: INFO + tags: [container, cis] + - macro: sensitive_mount + condition: (container.mount.dest[/proc*] != "N/A" or + container.mount.dest[/var/run/docker.sock] != "N/A" or + container.mount.dest[/] != "N/A" or + container.mount.dest[/etc] != "N/A" or + container.mount.dest[/root*] != "N/A") + - macro: container_entrypoint + condition: (not proc.pname exists or proc.pname in (runc:[0:PARENT], runc:[1:CHILD], docker-runc, exe)) + - rule: Launch Sensitive Mount Container + desc: > + Detect the initial process started by a container that has a mount from a sensitive host directory + (i.e. /proc). Exceptions are made for known trusted images. + condition: > + evt.type=execve and proc.vpid=1 and container + and sensitive_mount + and not trusted_containers + and not user_sensitive_mount_containers + output: Container with sensitive mount started (user=%user.name command=%proc.cmdline %container.info image=%container.image mounts=%container.mounts) + priority: INFO + tags: [container, cis] + - macro: allowed_containers + condition: (proc.vpid=1) + - rule: Launch Disallowed Container + desc: > + Detect the initial process started by a container that is not in a list of allowed containers. + condition: evt.type=execve and proc.vpid=1 and container and not allowed_containers + output: Container started and not in allowed list (user=%user.name command=%proc.cmdline %container.info image=%container.image) + priority: WARNING + tags: [container] + - rule: System user interactive + desc: an attempt to run interactive commands by a system (i.e. non-login) user + condition: spawned_process and system_users and interactive + output: "System user ran an interactive command (user=%user.name command=%proc.cmdline)" + priority: INFO + tags: [users] + - rule: Terminal shell in container + desc: A shell was used as the entrypoint/exec point into a container with an attached terminal. + condition: > + spawned_process and container + and shell_procs and proc.tty != 0 + and container_entrypoint + output: > + A shell was spawned in a container with an attached terminal (user=%user.name %container.info + shell=%proc.name parent=%proc.pname cmdline=%proc.cmdline terminal=%proc.tty) + priority: NOTICE + tags: [container, shell] + - list: known_shell_spawn_cmdlines + items: [ + '"sh -c uname -p 2> /dev/null"', + '"sh -c uname -s 2>&1"', + '"sh -c uname -r 2>&1"', + '"sh -c uname -v 2>&1"', + '"sh -c uname -a 2>&1"', + '"sh -c ruby -v 2>&1"', + '"sh -c getconf CLK_TCK"', + '"sh -c getconf PAGESIZE"', + '"sh -c LC_ALL=C LANG=C /sbin/ldconfig -p 2>/dev/null"', + '"sh -c LANG=C /sbin/ldconfig -p 2>/dev/null"', + '"sh -c /sbin/ldconfig -p 2>/dev/null"', + '"sh -c stty -a 2>/dev/null"', + '"sh -c stty -a < /dev/tty"', + '"sh -c stty -g < /dev/tty"', + '"sh -c node index.js"', + '"sh -c node index"', + '"sh -c node ./src/start.js"', + '"sh -c node app.js"', + '"sh -c node -e \"require(''nan'')\""', + '"sh -c node -e \"require(''nan'')\")"', + '"sh -c node $NODE_DEBUG_OPTION index.js "', + '"sh -c crontab -l 2"', + '"sh -c lsb_release -a"', + '"sh -c lsb_release -is 2>/dev/null"', + '"sh -c whoami"', + '"sh -c node_modules/.bin/bower-installer"', + '"sh -c /bin/hostname -f 2> /dev/null"', + '"sh -c locale -a"', + '"sh -c -t -i"', + '"sh -c openssl version"', + '"bash -c id -Gn kafadmin"', + '"sh -c /bin/sh -c ''date +%%s''"' + ] + - list: user_known_shell_spawn_binaries + items: [] + - macro: user_shell_container_exclusions + condition: (never_true) + - macro: login_doing_dns_lookup + condition: (proc.name=login and fd.l4proto=udp and fd.sport=53) + - rule: System procs network activity + desc: any network activity performed by system binaries that are not expected to send or receive any network traffic + condition: > + (fd.sockfamily = ip and system_procs) + and (inbound_outbound) + and not proc.name in (systemd, hostid, id) + and not login_doing_dns_lookup + output: > + Known system binary sent/received network traffic + (user=%user.name command=%proc.cmdline connection=%fd.name) + priority: NOTICE + tags: [network] + - list: openvpn_udp_ports + items: [1194, 1197, 1198, 8080, 9201] + - list: l2tp_udp_ports + items: [500, 1701, 4500, 10000] + - list: statsd_ports + items: [8125] + - list: ntp_ports + items: [123] + - list: test_connect_ports + items: [0, 9, 80, 3306] + - macro: do_unexpected_udp_check + condition: (never_true) + - list: expected_udp_ports + items: [53, openvpn_udp_ports, l2tp_udp_ports, statsd_ports, ntp_ports, test_connect_ports] + - macro: expected_udp_traffic + condition: fd.port in (expected_udp_ports) + - rule: Unexpected UDP Traffic + desc: UDP traffic not on port 53 (DNS) or other commonly used ports + condition: (inbound_outbound) and do_unexpected_udp_check and fd.l4proto=udp and not expected_udp_traffic + output: > + Unexpected UDP Traffic Seen + (user=%user.name command=%proc.cmdline connection=%fd.name proto=%fd.l4proto evt=%evt.type %evt.args) + priority: NOTICE + tags: [network] + - macro: somebody_becoming_themself + condition: ((user.name=nobody and evt.arg.uid=nobody) or + (user.name=www-data and evt.arg.uid=www-data) or + (user.name=_apt and evt.arg.uid=_apt) or + (user.name=postfix and evt.arg.uid=postfix) or + (user.name=pki-agent and evt.arg.uid=pki-agent) or + (user.name=pki-acme and evt.arg.uid=pki-acme) or + (user.name=nfsnobody and evt.arg.uid=nfsnobody) or + (user.name=postgres and evt.arg.uid=postgres)) + - macro: nrpe_becoming_nagios + condition: (proc.name=nrpe and evt.arg.uid=nagios) + - macro: known_user_in_container + condition: (container and user.name != "N/A") + - rule: Non sudo setuid + desc: > + an attempt to change users by calling setuid. sudo/su are excluded. users "root" and "nobody" + suing to itself are also excluded, as setuid calls typically involve dropping privileges. + condition: > + evt.type=setuid and evt.dir=> + and (known_user_in_container or not container) + and not user.name=root and not somebody_becoming_themself + and not proc.name in (known_setuid_binaries, userexec_binaries, mail_binaries, docker_binaries, + nomachine_binaries) + and not java_running_sdjagent + and not nrpe_becoming_nagios + output: > + Unexpected setuid call by non-sudo, non-root program (user=%user.name cur_uid=%user.uid parent=%proc.pname + command=%proc.cmdline uid=%evt.arg.uid) + priority: NOTICE + tags: [users] + - rule: User mgmt binaries + desc: > + activity by any programs that can manage users, passwords, or permissions. sudo and su are excluded. + Activity in containers is also excluded--some containers create custom users on top + of a base linux distribution at startup. + Some innocuous commandlines that don't actually change anything are excluded. + condition: > + spawned_process and proc.name in (user_mgmt_binaries) and + not proc.name in (su, sudo, lastlog, nologin, unix_chkpwd) and not container and + not proc.pname in (cron_binaries, systemd, systemd.postins, udev.postinst, run-parts) and + not proc.cmdline startswith "passwd -S" and + not proc.cmdline startswith "useradd -D" and + not proc.cmdline startswith "systemd --version" and + not run_by_qualys and + not run_by_sumologic_securefiles and + not run_by_yum and + not run_by_ms_oms and + not run_by_google_accounts_daemon + output: > + User management binary command run outside of container + (user=%user.name command=%proc.cmdline parent=%proc.pname gparent=%proc.aname[2] ggparent=%proc.aname[3] gggparent=%proc.aname[4]) + priority: NOTICE + tags: [host, users] + - list: allowed_dev_files + items: [ + /dev/null, /dev/stdin, /dev/stdout, /dev/stderr, + /dev/random, /dev/urandom, /dev/console, /dev/kmsg + ] + - rule: Create files below dev + desc: creating any files below /dev other than known programs that manage devices. Some rootkits hide files in /dev. + condition: > + fd.directory = /dev and + (evt.type = creat or (evt.type = open and evt.arg.flags contains O_CREAT)) + and not proc.name in (dev_creation_binaries) + and not fd.name in (allowed_dev_files) + and not fd.name startswith /dev/tty + output: "File created below /dev by untrusted program (user=%user.name command=%proc.cmdline file=%fd.name)" + priority: ERROR + tags: [filesystem] + - macro: ec2_metadata_containers + condition: container + - rule: Contact EC2 Instance Metadata Service From Container + desc: Detect attempts to contact the EC2 Instance Metadata Service from a container + condition: outbound and fd.sip="169.254.169.254" and container and not ec2_metadata_containers + output: Outbound connection to EC2 instance metadata service (command=%proc.cmdline connection=%fd.name %container.info image=%container.image) + priority: NOTICE + tags: [network, aws, container] + - macro: k8s_api_server + condition: (fd.sip="1.2.3.4" and fd.sport=8080) + - macro: k8s_containers + condition: > + (container.image startswith registry.k8s.io/hyperkube-amd64 or + container.image startswith registry.k8s.io/kube2sky or + container.image startswith sysdig/agent or + container.image startswith sysdig/falco or + container.image startswith sysdig/sysdig) + - rule: Contact K8S API Server From Container + desc: Detect attempts to contact the K8S API Server from a container + condition: outbound and k8s_api_server and container and not k8s_containers + output: Unexpected connection to K8s API Server from container (command=%proc.cmdline %container.info image=%container.image connection=%fd.name) + priority: NOTICE + tags: [network, k8s, container] + - macro: nodeport_containers + condition: container + - rule: Unexpected K8s NodePort Connection + desc: Detect attempts to use K8s NodePorts from a container + condition: (inbound_outbound) and fd.sport >= 30000 and fd.sport <= 32767 and container and not nodeport_containers + output: Unexpected K8s NodePort Connection (command=%proc.cmdline connection=%fd.name) + priority: NOTICE + tags: [network, k8s, container] + + falco_rules_local: | + #################### + # Your custom rules! + #################### + + # Add new rules, like this one + # - rule: The program "sudo" is run in a container + # desc: An event will trigger every time you run sudo in a container + # condition: evt.type = execve and evt.dir=< and container.id != host and proc.name = sudo + # output: "Sudo run in container (user=%user.name %container.info parent=%proc.pname cmdline=%proc.cmdline)" + # priority: ERROR + # tags: [users, container] + + # Or override/append to any rule, macro, or list from the Default Rules + custom_rules: {} + # Although Falco comes with a nice default rule set for detecting weird + # behavior in containers, our users are going to customize the run-time + # security rule sets or policies for the specific container images and + # applications they run. This feature can be handled in this section. + # + # Example: + # + # rules-traefik.yaml: |- + # [ rule body ] + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - k8sksauth-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + falco: + services: null + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + +manifests: + daemonset: true + configmap_etc: true + configmap_custom_rules: false + configmap_bin: true + secret_registry: true +... diff --git a/flannel/Chart.yaml b/flannel/Chart.yaml new file mode 100644 index 0000000000..7ec5a5f815 --- /dev/null +++ b/flannel/Chart.yaml @@ -0,0 +1,30 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: v0.8.0 +description: OpenStack-Helm BootStrap Flannel +name: flannel +version: 2024.2.0 +home: https://github.com/coreos/flannel +icon: https://raw.githubusercontent.com/coreos/flannel/master/logos/flannel-horizontal-color.png +sources: + - https://github.com/coreos/flannel + - https://opendev.org/openstack/openstack-helm +maintainers: + - name: OpenStack-Helm Authors +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit + version: ">= 0.1.0" +... diff --git a/flannel/templates/configmap-bin.yaml b/flannel/templates/configmap-bin.yaml new file mode 100644 index 0000000000..6523a95147 --- /dev/null +++ b/flannel/templates/configmap-bin.yaml @@ -0,0 +1,25 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_bin }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: flannel-bin +data: + image-repo-sync.sh: | +{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} +{{- end }} diff --git a/flannel/templates/configmap-kube-flannel-cfg.yaml b/flannel/templates/configmap-kube-flannel-cfg.yaml new file mode 100644 index 0000000000..2e852dc3b5 --- /dev/null +++ b/flannel/templates/configmap-kube-flannel-cfg.yaml @@ -0,0 +1,41 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_kube_flannel_cfg }} +{{- $envAll := . }} +--- +kind: ConfigMap +apiVersion: v1 +metadata: + name: kube-flannel-cfg + labels: + tier: node + app: flannel +data: + cni-conf.json: | + { + "name": "cbr0", + "type": "flannel", + "delegate": { + "isDefaultGateway": true + } + } + net-conf.json: | + { + "Network": "{{ .Values.networking.podSubnet }}", + "Backend": { + "Type": "vxlan" + } + } +{{- end }} diff --git a/flannel/templates/daemonset-kube-flannel-ds.yaml b/flannel/templates/daemonset-kube-flannel-ds.yaml new file mode 100644 index 0000000000..92cb94ff79 --- /dev/null +++ b/flannel/templates/daemonset-kube-flannel-ds.yaml @@ -0,0 +1,142 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.daemonset_kube_flannel_ds }} +{{- $envAll := . }} + +{{- $serviceAccountName := printf "%s-%s" .Release.Name "flannel" }} +{{ tuple $envAll "flannel" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ $serviceAccountName }} +rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - apiGroups: + - "" + resources: + - nodes + verbs: + - list + - watch + - apiGroups: + - "" + resources: + - nodes/status + verbs: + - patch +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ $serviceAccountName }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ $serviceAccountName }} +subjects: +- kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: kube-flannel-ds + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: + tier: node + app: flannel +{{ tuple $envAll "flannel" "node" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + selector: + matchLabels: + tier: node + app: flannel +{{ tuple $envAll "flannel" "node" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} + template: + metadata: + labels: + tier: node + app: flannel +{{ tuple $envAll "flannel" "node" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + spec: + hostNetwork: true + nodeSelector: + beta.kubernetes.io/arch: amd64 + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + serviceAccountName: {{ $serviceAccountName }} + initContainers: +{{ tuple $envAll "flannel" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: kube-flannel +{{ tuple $envAll "flannel" | include "helm-toolkit.snippets.image" | indent 10 }} + securityContext: + privileged: true + command: ["/opt/bin/flanneld", "--ip-masq", "--kube-subnet-mgr"] + env: + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: run + mountPath: /run + - name: flannel-cfg + mountPath: /etc/kube-flannel/ + - name: install-cni + image: {{ .Values.images.tags.flannel }} + command: ["/bin/sh", "-c", "set -e -x; cp -f /etc/kube-flannel/cni-conf.json /etc/cni/net.d/10-flannel.conf; while true; do sleep 3600; done"] + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: cni + mountPath: /etc/cni/net.d + - name: flannel-cfg + mountPath: /etc/kube-flannel/ + volumes: + - name: pod-tmp + emptyDir: {} + - name: run + hostPath: + path: /run + - name: cni + hostPath: + path: /etc/cni/net.d + - name: flannel-cfg + configMap: + name: kube-flannel-cfg +{{- end }} diff --git a/flannel/templates/job-image-repo-sync.yaml b/flannel/templates/job-image-repo-sync.yaml new file mode 100644 index 0000000000..82e74271c6 --- /dev/null +++ b/flannel/templates/job-image-repo-sync.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "flannel" -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/flannel/templates/secret-registry.yaml b/flannel/templates/secret-registry.yaml new file mode 100644 index 0000000000..da979b3223 --- /dev/null +++ b/flannel/templates/secret-registry.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_registry .Values.endpoints.oci_image_registry.auth.enabled }} +{{ include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) }} +{{- end }} diff --git a/flannel/values.yaml b/flannel/values.yaml new file mode 100644 index 0000000000..28671df2c6 --- /dev/null +++ b/flannel/values.yaml @@ -0,0 +1,106 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# https://raw.githubusercontent.com/coreos/flannel/v0.8.0/Documentation/kube-flannel.yml + +--- +labels: + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +images: + tags: + flannel: quay.io/coreos/flannel:v0.8.0-amd64 + dep_check: quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal + image_repo_sync: docker.io/library/docker:17.07.0 + pull_policy: IfNotPresent + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + - flannel + +pod: + resources: + enabled: false + jobs: + image_repo_sync: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + +networking: + podSubnet: 192.168.0.0/16 + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - flannel-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + flannel: + services: null + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + +secrets: + oci_image_registry: + flannel: flannel-oci-image-registry-key + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + flannel: + username: flannel + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + +manifests: + configmap_bin: true + configmap_kube_flannel_cfg: true + daemonset_kube_flannel_ds: true + job_image_repo_sync: true + secret_registry: true +... diff --git a/fluentbit/Chart.yaml b/fluentbit/Chart.yaml new file mode 100644 index 0000000000..1d79e68c83 --- /dev/null +++ b/fluentbit/Chart.yaml @@ -0,0 +1,29 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: v0.14.2 +description: OpenStack-Helm Fluentbit +name: fluentbit +version: 2024.2.0 +home: https://www.fluentbit.io/ +sources: + - https://github.com/fluent/fluentbit + - https://opendev.org/openstack/openstack-helm-infra +maintainers: + - name: OpenStack-Helm Authors +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit/ + version: ">= 0.1.0" +... diff --git a/fluentbit/templates/bin/_fluent-bit.sh.tpl b/fluentbit/templates/bin/_fluent-bit.sh.tpl new file mode 100644 index 0000000000..613c99d1fa --- /dev/null +++ b/fluentbit/templates/bin/_fluent-bit.sh.tpl @@ -0,0 +1,25 @@ +#!/bin/sh + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +if [ -d "/var/log/journal" ]; then + export JOURNAL_PATH="/var/log/journal" +else + export JOURNAL_PATH="/run/log/journal" +fi + +exec /fluent-bit/bin/fluent-bit -c /fluent-bit/etc/fluent-bit.conf diff --git a/fluentbit/templates/configmap-bin.yaml b/fluentbit/templates/configmap-bin.yaml new file mode 100644 index 0000000000..11bb1a065e --- /dev/null +++ b/fluentbit/templates/configmap-bin.yaml @@ -0,0 +1,27 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_bin }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: fluentbit-bin +data: + fluent-bit.sh: | +{{ tuple "bin/_fluent-bit.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + image-repo-sync.sh: | +{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} +{{- end }} diff --git a/fluentbit/templates/configmap-etc.yaml b/fluentbit/templates/configmap-etc.yaml new file mode 100644 index 0000000000..501f44898d --- /dev/null +++ b/fluentbit/templates/configmap-etc.yaml @@ -0,0 +1,26 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_etc }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: fluentbit-etc +type: Opaque +data: + fluent-bit.conf: {{ .Values.conf.fluentbit.template | b64enc }} + parsers.conf: {{ .Values.conf.parsers.template | b64enc }} +{{- end }} diff --git a/fluentbit/templates/daemonset-fluent-bit.yaml b/fluentbit/templates/daemonset-fluent-bit.yaml new file mode 100644 index 0000000000..755f7abcad --- /dev/null +++ b/fluentbit/templates/daemonset-fluent-bit.yaml @@ -0,0 +1,154 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.daemonset_fluentbit }} +{{- $envAll := . }} + +{{- $mounts_fluentbit := .Values.pod.mounts.fluentbit.fluentbit }} + +{{- $serviceAccountName := printf "%s-%s" .Release.Name "fluentbit" }} +{{ tuple $envAll "fluentbit" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ $serviceAccountName }} + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ $serviceAccountName }} +rules: + - apiGroups: + - "" + resources: + - namespaces + - nodes + - pods + - services + - replicationcontrollers + - limitranges + verbs: + - get + - list + - watch + - apiGroups: + - apps + resources: + - statefulsets + - daemonsets + - deployments + - replicasets + verbs: + - get + - list + - watch +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: fluentbit + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "fluentbit" "daemon" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + selector: + matchLabels: +{{ tuple $envAll "fluentbit" "daemon" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} +{{ tuple $envAll "fluentbit" | include "helm-toolkit.snippets.kubernetes_upgrades_daemonset" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "fluentbit" "daemon" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} +{{ dict "envAll" $envAll "podName" "fluentbit" "containerNames" (list "fluentbit") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "fluentbit" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} +{{ if $envAll.Values.pod.tolerations.fluentbit.enabled }} +{{ tuple $envAll "fluentbit" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }} +{{ else }} + nodeSelector: + {{ .Values.labels.fluentbit.node_selector_key }}: {{ .Values.labels.fluentbit.node_selector_value | quote }} +{{ end }} + hostNetwork: true + hostPID: true + dnsPolicy: {{ .Values.pod.dns_policy }} + initContainers: +{{ tuple $envAll "fluentbit" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: fluentbit +{{ tuple $envAll "fluentbit" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.fluentbit | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "fluentbit" "container" "fluentbit" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/fluent-bit.sh + env: + - name: FLUENTD_HOST + value: {{ tuple "fluentd" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" | quote }} + - name: FLUENTD_PORT + value: {{ tuple "fluentd" "internal" "service" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: fluentbit-bin + mountPath: /tmp/fluent-bit.sh + subPath: fluent-bit.sh + readOnly: true + - name: varlog + mountPath: /var/log + readOnly: true + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + readOnly: true + - name: fluentbit-etc + mountPath: /fluent-bit/etc/fluent-bit.conf + subPath: fluent-bit.conf + readOnly: true + - name: fluentbit-etc + mountPath: /fluent-bit/etc/parsers.conf + subPath: parsers.conf + readOnly: true +{{ if $mounts_fluentbit.volumeMounts }}{{ toYaml $mounts_fluentbit.volumeMounts | indent 12 }}{{ end }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: varlog + hostPath: + path: /var/log + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers + - name: fluentbit-bin + configMap: + name: fluentbit-bin + defaultMode: 0555 + - name: fluentbit-etc + secret: + secretName: fluentbit-etc + defaultMode: 0444 +{{ if $mounts_fluentbit.volumes }}{{ toYaml $mounts_fluentbit.volumes | indent 8 }}{{ end }} +{{- end }} diff --git a/fluentbit/templates/job-image-repo-sync.yaml b/fluentbit/templates/job-image-repo-sync.yaml new file mode 100644 index 0000000000..1d4ff27fa1 --- /dev/null +++ b/fluentbit/templates/job-image-repo-sync.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "fluentbit" -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/fluentbit/templates/secret-registry.yaml b/fluentbit/templates/secret-registry.yaml new file mode 100644 index 0000000000..da979b3223 --- /dev/null +++ b/fluentbit/templates/secret-registry.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_registry .Values.endpoints.oci_image_registry.auth.enabled }} +{{ include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) }} +{{- end }} diff --git a/fluentbit/values.yaml b/fluentbit/values.yaml new file mode 100644 index 0000000000..a98b6be66b --- /dev/null +++ b/fluentbit/values.yaml @@ -0,0 +1,279 @@ +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for fluentbit +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +--- +release_group: null + +labels: + fluentbit: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +images: + tags: + fluentbit: docker.io/fluent/fluent-bit:0.14.2 + dep_check: quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal + image_repo_sync: docker.io/library/docker:17.07.0 + pull_policy: IfNotPresent + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - fluentbit-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + +conf: + fluentbit: + template: | + [SERVICE] + Daemon false + Flush 30 + Log_Level info + Parsers_File parsers.conf + + [INPUT] + Buffer_Chunk_Size 1M + Buffer_Max_Size 1M + Mem_Buf_Limit 5MB + Name tail + Path /var/log/kern.log + Tag kernel + + [INPUT] + Buffer_Chunk_Size 1M + Buffer_Max_Size 1M + Mem_Buf_Limit 5MB + Name tail + Parser docker + Path /var/log/containers/*.log + Tag kube.* + + [INPUT] + Buffer_Chunk_Size 1M + Buffer_Max_Size 1M + Mem_Buf_Limit 5MB + Name tail + Path /var/log/libvirt/libvirtd.log + Tag libvirt + + [INPUT] + Buffer_Chunk_Size 1M + Buffer_Max_Size 1M + Mem_Buf_Limit 5MB + Name tail + Path /var/log/libvirt/qemu/*.log + Tag qemu + + [INPUT] + Buffer_Chunk_Size 1M + Buffer_Max_Size 1M + Mem_Buf_Limit 5MB + Name systemd + Path ${JOURNAL_PATH} + Systemd_Filter _SYSTEMD_UNIT=kubelet.service + Tag journal.* + + [INPUT] + Buffer_Chunk_Size 1M + Buffer_Max_Size 1M + Mem_Buf_Limit 5MB + Name systemd + Path ${JOURNAL_PATH} + Systemd_Filter _SYSTEMD_UNIT=docker.service + Tag journal.* + + [FILTER] + Interval 1s + Match ** + Name throttle + Rate 1000 + Window 300 + + [FILTER] + Match libvirt + Name record_modifier + Record hostname ${HOSTNAME} + + [FILTER] + Match qemu + Name record_modifier + Record hostname ${HOSTNAME} + + [FILTER] + Match kernel + Name record_modifier + Record hostname ${HOSTNAME} + + [FILTER] + Match journal.** + Name modify + Rename _BOOT_ID BOOT_ID + Rename _CAP_EFFECTIVE CAP_EFFECTIVE + Rename _CMDLINE CMDLINE + Rename _COMM COMM + Rename _EXE EXE + Rename _GID GID + Rename _HOSTNAME HOSTNAME + Rename _MACHINE_ID MACHINE_ID + Rename _PID PID + Rename _SYSTEMD_CGROUP SYSTEMD_CGROUP + Rename _SYSTEMD_SLICE SYSTEMD_SLICE + Rename _SYSTEMD_UNIT SYSTEMD_UNIT + Rename _TRANSPORT TRANSPORT + Rename _UID UID + + [OUTPUT] + Match **.fluentd** + Name null + + [FILTER] + Match kube.* + Merge_JSON_Log true + Name kubernetes + + [OUTPUT] + Host ${FLUENTD_HOST} + Match * + Name forward + Port ${FLUENTD_PORT} + parsers: + template: | + [PARSER] + Decode_Field_As escaped_utf8 log + Format json + Name docker + Time_Format %Y-%m-%dT%H:%M:%S.%L + Time_Keep true + Time_Key time + +secrets: + oci_image_registry: + fluentbit: fluentbit-oci-image-registry-key + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + fluentbit: + username: fluentbit + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + fluentd: + namespace: null + name: fluentd + hosts: + default: fluentd-logging + host_fqdn_override: + default: null + path: + default: null + scheme: + default: http + port: + service: + default: 24224 + metrics: + default: 24220 + +pod: + security_context: + fluentbit: + pod: + runAsUser: 65534 + container: + fluentbit: + runAsUser: 0 + readOnlyRootFilesystem: false + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + dns_policy: "ClusterFirstWithHostNet" + lifecycle: + upgrades: + daemonsets: + pod_replacement_strategy: RollingUpdate + fluentbit: + enabled: true + min_ready_seconds: 0 + max_unavailable: 1 + resources: + enabled: false + fluentbit: + limits: + memory: '400Mi' + cpu: '400m' + requests: + memory: '100Mi' + cpu: '100m' + tolerations: + fluentbit: + enabled: false + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + - key: node-role.kubernetes.io/control-plane + operator: Exists + - key: node-role.kubernetes.io/node + operator: Exists + mounts: + fluentbit: + fluentbit: + +manifests: + configmap_bin: true + configmap_etc: true + daemonset_fluentbit: true + job_image_repo_sync: true + secret_registry: true +... diff --git a/fluentd/Chart.yaml b/fluentd/Chart.yaml new file mode 100644 index 0000000000..55b52e5bab --- /dev/null +++ b/fluentd/Chart.yaml @@ -0,0 +1,29 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: v1.10.1 +description: OpenStack-Helm Fluentd +name: fluentd +version: 2024.2.0 +home: https://www.fluentd.org/ +sources: + - https://github.com/fluent/fluentd + - https://opendev.org/openstack/openstack-helm-infra +maintainers: + - name: OpenStack-Helm Authors +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit/ + version: ">= 0.1.0" +... diff --git a/fluentd/templates/bin/_fluentd.sh.tpl b/fluentd/templates/bin/_fluentd.sh.tpl new file mode 100644 index 0000000000..56e848e04d --- /dev/null +++ b/fluentd/templates/bin/_fluentd.sh.tpl @@ -0,0 +1,29 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +COMMAND="${@:-start}" + +function start () { + chmod 1777 /tmp + exec fluentd -c /fluentd/etc/main.conf +} + +function stop () { + kill -TERM 1 +} + +$COMMAND diff --git a/fluentd/templates/configmap-bin.yaml b/fluentd/templates/configmap-bin.yaml new file mode 100644 index 0000000000..f258605b05 --- /dev/null +++ b/fluentd/templates/configmap-bin.yaml @@ -0,0 +1,27 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_bin }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ printf "%s-%s" $envAll.Release.Name "fluentd-bin" | quote }} +data: + fluentd.sh: | +{{ tuple "bin/_fluentd.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + image-repo-sync.sh: | +{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} +{{- end }} diff --git a/fluentd/templates/configmap-etc.yaml b/fluentd/templates/configmap-etc.yaml new file mode 100644 index 0000000000..81c1125857 --- /dev/null +++ b/fluentd/templates/configmap-etc.yaml @@ -0,0 +1,40 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "fluentd_main" }} +{{- $path := .Values.conf.fluentd.path}} +{{- range $name, $conf := .Values.conf.fluentd.conf }} +{{ printf "%s %s/%s.conf" "@include" $path $name | indent 4}} +{{- end }} +{{- end }} + +{{- if .Values.manifests.configmap_etc }} +{{ $envAll := .}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ printf "%s-%s" $envAll.Release.Name "fluentd-etc" | quote }} + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +type: Opaque +stringData: + main.conf: | +{{- template "fluentd_main" . }} +data: +{{- range $name, $config := .Values.conf.fluentd.conf }} +{{- $filename := printf "%s.conf" $name}} +{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" $config "key" $filename "format" "Secret") | indent 2 }} +{{- end }} +{{- end }} diff --git a/fluentd/templates/daemonset.yaml b/fluentd/templates/daemonset.yaml new file mode 100644 index 0000000000..7ddbf6a218 --- /dev/null +++ b/fluentd/templates/daemonset.yaml @@ -0,0 +1,228 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "probeTemplate" }} +tcpSocket: + port: {{ tuple "fluentd" "internal" "service" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +{{- end }} + +{{- if .Values.manifests.daemonset }} +{{- $envAll := . }} + +{{- $config_path := .Values.conf.fluentd.path }} +{{- $mounts_fluentd := .Values.pod.mounts.fluentd.fluentd }} +{{- $prometheus_annotations := $envAll.Values.monitoring.prometheus.fluentd }} + +{{- $kafkaBroker := tuple "kafka" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }} +{{- $kafkaBrokerPort := tuple "kafka" "internal" "broker" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +{{- $kafkaBrokerURI := printf "%s" $kafkaBroker }} + +{{- $rcControllerName := printf "%s-%s" $envAll.Release.Name "fluentd" }} +{{ tuple $envAll "fluentd" $rcControllerName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ $rcControllerName | quote }} +subjects: + - kind: ServiceAccount + name: {{ $rcControllerName | quote }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ $rcControllerName | quote }} + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ $rcControllerName | quote }} +rules: + - apiGroups: + - "" + resources: + - namespaces + - nodes + - pods + - services + - replicationcontrollers + - limitranges + verbs: + - get + - list + - watch + - apiGroups: + - apps + resources: + - statefulsets + - daemonsets + - deployments + - replicasets + verbs: + - get + - list + - watch +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: {{ $rcControllerName | quote }} + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "fluentd" "internal" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: +{{ tuple $envAll "fluentd" | include "helm-toolkit.snippets.kubernetes_upgrades_daemonset" | indent 2 }} + selector: + matchLabels: +{{ tuple $envAll "fluentd" "internal" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} + template: + metadata: + labels: +{{ tuple $envAll "fluentd" "internal" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} +{{- if .Values.monitoring.prometheus.enabled }} +{{ tuple $prometheus_annotations | include "helm-toolkit.snippets.prometheus_pod_annotations" | indent 8 }} +{{- end }} +{{ dict "envAll" $envAll "podName" "fluentd" "containerNames" (list "fluentd" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "fluentd" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $rcControllerName | quote }} +{{ if $envAll.Values.pod.tolerations.fluentd.enabled }} +{{ tuple $envAll "fluentd" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }} +{{ end }} + nodeSelector: + {{ .Values.labels.fluentd.node_selector_key }}: {{ .Values.labels.fluentd.node_selector_value | quote }} + terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.fluentd.timeout | default "30" }} + initContainers: +{{ tuple $envAll "fluentd" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: fluentd +{{ tuple $envAll "fluentd" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.fluentd | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "fluentd" "container" "fluentd" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/fluentd.sh + - start + ports: + - name: forward + containerPort: {{ tuple "fluentd" "internal" "service" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - name: metrics + containerPort: {{ tuple "fluentd" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +{{ dict "envAll" . "component" "fluentd" "container" "fluentd" "type" "readiness" "probeTemplate" (include "probeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }} +{{ dict "envAll" . "component" "fluentd" "container" "fluentd" "type" "liveness" "probeTemplate" (include "probeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }} + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: FLUENTD_PORT + value: {{ tuple "fluentd" "internal" "service" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: ELASTICSEARCH_HOST + value: {{ tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" | quote }} + - name: ELASTICSEARCH_PORT + value: {{ tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: ELASTICSEARCH_SCHEME + value: {{ tuple "elasticsearch" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" | quote }} + - name: KAFKA_BROKER + value: {{ $kafkaBrokerURI }} +{{- if .Values.pod.env.fluentd.vars }} +{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.fluentd.vars | indent 12 }} +{{- end }} +{{- if .Values.pod.env.fluentd.secrets }} +{{ tuple $envAll .Values.pod.env.fluentd.secrets | include "helm-toolkit.utils.to_k8s_env_secret_vars" | indent 12 }} +{{- end }} + - name: ELASTICSEARCH_USERNAME + valueFrom: + secretKeyRef: + name: {{ printf "%s-%s" $envAll.Release.Name "elasticsearch-user" | quote }} + key: ELASTICSEARCH_USERNAME + - name: ELASTICSEARCH_PASSWORD + valueFrom: + secretKeyRef: + name: {{ printf "%s-%s" $envAll.Release.Name "elasticsearch-user" | quote }} + key: ELASTICSEARCH_PASSWORD +{{- if .Values.manifests.secret_kafka }} + - name: KAFKA_USERNAME + valueFrom: + secretKeyRef: + name: {{ printf "%s-%s" $envAll.Release.Name "kafka-user" | quote }} + key: KAFKA_USERNAME + - name: KAFKA_PASSWORD + valueFrom: + secretKeyRef: + name: {{ printf "%s-%s" $envAll.Release.Name "kafka-user" | quote }} + key: KAFKA_PASSWORD +{{- end }} + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: varlog + mountPath: /var/log + - name: varlibdockercontainers + mountPath: /var/lib/docker/containers + readOnly: true + - name: pod-etc-fluentd + mountPath: /fluentd/etc + - name: fluentd-etc + mountPath: {{ printf "%s/%s.conf" $config_path "main" }} + subPath: {{ printf "%s.conf" "main"}} + readOnly: true +{{- range $name, $config := .Values.conf.fluentd.conf }} + - name: fluentd-etc + mountPath: {{ printf "%s/%s.conf" $config_path $name }} + subPath: {{ printf "%s.conf" $name }} + readOnly: true +{{- end }} + - name: fluentd-bin + mountPath: /tmp/fluentd.sh + subPath: fluentd.sh + readOnly: true +{{- dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.endpoints.elasticsearch.auth.admin.secret.tls.internal "path" "/etc/elasticsearch/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }} +{{ if $mounts_fluentd.volumeMounts }}{{ toYaml $mounts_fluentd.volumeMounts | indent 12 }}{{- end }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: varlog + hostPath: + path: /var/log + - name: varlibdockercontainers + hostPath: + path: /var/lib/docker/containers + - name: pod-etc-fluentd + emptyDir: {} +{{ if and (.Values.manifests.secret_fluentd_env) (.Values.pod.env.fluentd.secrets) }} + - name: {{ printf "%s-%s" $envAll.Release.Name "env-secret" | quote }} + secret: + secretName: {{ printf "%s-%s" $envAll.Release.Name "env-secret" | quote }} + defaultMode: 0444 +{{- end }} + - name: fluentd-etc + secret: + secretName: {{ printf "%s-%s" $envAll.Release.Name "fluentd-etc" | quote }} + defaultMode: 0444 + - name: fluentd-bin + configMap: + name: {{ printf "%s-%s" $envAll.Release.Name "fluentd-bin" | quote }} + defaultMode: 0555 +{{- dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.endpoints.elasticsearch.auth.admin.secret.tls.internal | include "helm-toolkit.snippets.tls_volume" | indent 8 }} +{{ if $mounts_fluentd.volumes }}{{ toYaml $mounts_fluentd.volumes | indent 8 }}{{- end }} +{{- end }} diff --git a/fluentd/templates/job-image-repo-sync.yaml b/fluentd/templates/job-image-repo-sync.yaml new file mode 100644 index 0000000000..2e841e8297 --- /dev/null +++ b/fluentd/templates/job-image-repo-sync.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "fluentd" -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/fluentd/templates/network_policy.yaml b/fluentd/templates/network_policy.yaml new file mode 100644 index 0000000000..771b946e8e --- /dev/null +++ b/fluentd/templates/network_policy.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */}} + +{{- if .Values.manifests.network_policy -}} +{{ $netpol_opts := dict "envAll" . "name" "application" "label" "fluentd" }} +{{ $netpol_opts | include "helm-toolkit.manifests.kubernetes_network_policy" }} +{{- end -}} diff --git a/fluentd/templates/secret-elasticsearch-creds.yaml b/fluentd/templates/secret-elasticsearch-creds.yaml new file mode 100644 index 0000000000..e20b78e911 --- /dev/null +++ b/fluentd/templates/secret-elasticsearch-creds.yaml @@ -0,0 +1,26 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_elasticsearch }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ printf "%s-%s" $envAll.Release.Name "elasticsearch-user" | quote }} +type: Opaque +data: + ELASTICSEARCH_USERNAME: {{ .Values.endpoints.elasticsearch.auth.admin.username | b64enc }} + ELASTICSEARCH_PASSWORD: {{ .Values.endpoints.elasticsearch.auth.admin.password | b64enc }} +{{- end }} diff --git a/fluentd/templates/secret-fluentd.yaml b/fluentd/templates/secret-fluentd.yaml new file mode 100644 index 0000000000..db4a9620e8 --- /dev/null +++ b/fluentd/templates/secret-fluentd.yaml @@ -0,0 +1,27 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and (.Values.manifests.secret_fluentd_env) (.Values.pod.env.fluentd.secrets) }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ printf "%s-%s" $envAll.Release.Name "env-secret" | quote }} +type: Opaque +data: + {{ range $key, $value := .Values.pod.env.fluentd.secrets }} + {{$key | upper}}: {{ $value | b64enc }} + {{- end }} +{{- end }} diff --git a/fluentd/templates/secret-kafka-creds.yaml b/fluentd/templates/secret-kafka-creds.yaml new file mode 100644 index 0000000000..e1ed094fb5 --- /dev/null +++ b/fluentd/templates/secret-kafka-creds.yaml @@ -0,0 +1,26 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_kafka }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ printf "%s-%s" $envAll.Release.Name "kafka-user" | quote }} +type: Opaque +data: + KAFKA_USERNAME: {{ .Values.endpoints.kafka.auth.admin.username | b64enc }} + KAFKA_PASSWORD: {{ .Values.endpoints.kafka.auth.admin.password | b64enc }} +{{- end }} diff --git a/fluentd/templates/secret-registry.yaml b/fluentd/templates/secret-registry.yaml new file mode 100644 index 0000000000..da979b3223 --- /dev/null +++ b/fluentd/templates/secret-registry.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_registry .Values.endpoints.oci_image_registry.auth.enabled }} +{{ include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) }} +{{- end }} diff --git a/fluentd/templates/service-fluentd.yaml b/fluentd/templates/service-fluentd.yaml new file mode 100644 index 0000000000..6d75de3eab --- /dev/null +++ b/fluentd/templates/service-fluentd.yaml @@ -0,0 +1,34 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.service_fluentd }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "fluentd" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +spec: + ports: + - name: forward + port: {{ tuple "fluentd" "internal" "service" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + {{ if .Values.network.fluentd.node_port.enabled }} + nodePort: {{ .Values.network.fluentd.node_port.port }} + {{ end }} + selector: +{{ tuple $envAll "fluentd" "internal" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + {{ if .Values.network.fluentd.node_port.enabled }} + type: NodePort + {{ end }} +{{- end }} diff --git a/fluentd/values.yaml b/fluentd/values.yaml new file mode 100644 index 0000000000..5249a777a3 --- /dev/null +++ b/fluentd/values.yaml @@ -0,0 +1,280 @@ +# Licensed under the Apache License, Version 2.0 (the 'License'); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an 'AS IS' BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for fluentd. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +--- +release_group: null + +labels: + fluentd: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +images: + tags: + fluentd: docker.io/openstackhelm/fluentd:latest-debian + dep_check: quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal + helm_tests: docker.io/openstackhelm/heat:wallaby-ubuntu_focal + image_repo_sync: docker.io/library/docker:17.07.0 + pull_policy: IfNotPresent + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - fluentd-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + fluentd: + services: null + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + +conf: + fluentd: + path: /fluentd/etc + conf: + input: | + + bind 0.0.0.0 + port "#{ENV['FLUENTD_PORT']}" + @type forward + + + + time_format %Y-%m-%dT%H:%M:%S.%NZ + @type json + + path /var/log/containers/*.log + read_from_head true + tag kubernetes.* + @type tail + + + @type relabel + @label @output + + output: | + + +secrets: + oci_image_registry: + fluentd: fluentd-oci-image-registry-key + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + fluentd: + username: fluentd + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + elasticsearch: + namespace: null + name: elasticsearch + auth: + admin: + username: admin + password: changeme + secret: + tls: + internal: elasticsearch-tls-api + hosts: + data: elasticsearch-data + default: elasticsearch-logging + discovery: elasticsearch-discovery + public: elasticsearch + host_fqdn_override: + default: null + path: + default: null + scheme: + default: http + port: + http: + default: 80 + fluentd: + namespace: null + name: fluentd + hosts: + default: fluentd-logging + host_fqdn_override: + default: null + path: + default: null + scheme: + default: http + port: + service: + default: 24224 + metrics: + default: 24231 + kafka: + namespace: null + name: kafka + auth: + admin: + username: admin + password: changeme + hosts: + default: kafka-broker + public: kafka + host_fqdn_override: + default: null + path: + default: null + scheme: + default: kafka + port: + broker: + default: 9092 + public: 80 + +monitoring: + prometheus: + enabled: true + fluentd: + scrape: true + port: 24231 + +network: + fluentd: + node_port: + enabled: false + port: 32329 + +network_policy: + fluentd: + ingress: + - {} + egress: + - {} + +pod: + env: + fluentd: + vars: null + secrets: null + tolerations: + fluentd: + enabled: false + security_context: + fluentd: + pod: + runAsUser: 0 + container: + fluentd: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + lifecycle: + upgrades: + daemonsets: + pod_replacement_strategy: RollingUpdate + fluentd: + enabled: true + min_ready_seconds: 0 + max_unavailable: 1 + termination_grace_period: + fluentd: + timeout: 30 + resources: + enabled: false + fluentd: + limits: + memory: '1024Mi' + cpu: '2000m' + requests: + memory: '128Mi' + cpu: '500m' + mounts: + fluentd: + fluentd: + probes: + fluentd: + fluentd: + readiness: + enabled: true + params: + initialDelaySeconds: 90 + timeoutSeconds: 30 + liveness: + enabled: true + params: + initialDelaySeconds: 180 + timeoutSeconds: 30 +manifests: + configmap_bin: true + configmap_etc: true + daemonset: true + job_image_repo_sync: true + network_policy: false + secret_elasticsearch: true + secret_fluentd_env: true + secret_kafka: false + secret_registry: true + service_fluentd: true +... diff --git a/gnocchi/.helmignore b/gnocchi/.helmignore new file mode 100644 index 0000000000..f0c1319444 --- /dev/null +++ b/gnocchi/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/gnocchi/Chart.yaml b/gnocchi/Chart.yaml new file mode 100644 index 0000000000..7a765ff550 --- /dev/null +++ b/gnocchi/Chart.yaml @@ -0,0 +1,30 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: v3.0.3 +description: OpenStack-Helm Gnocchi +name: gnocchi +version: 2024.2.0 +home: https://gnocchi.xyz/ +icon: https://gnocchi.xyz/_static/gnocchi-logo.png +sources: + - https://github.com/gnocchixyz/gnocchi + - https://opendev.org/openstack/openstack-helm +maintainers: + - name: OpenStack-Helm Authors +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit + version: ">= 0.1.0" +... diff --git a/gnocchi/templates/bin/_bootstrap.sh.tpl b/gnocchi/templates/bin/_bootstrap.sh.tpl new file mode 100644 index 0000000000..6452d0a073 --- /dev/null +++ b/gnocchi/templates/bin/_bootstrap.sh.tpl @@ -0,0 +1,18 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +{{ .Values.bootstrap.script | default "echo 'Not Enabled'" }} diff --git a/gnocchi/templates/bin/_ceph-admin-keyring.sh.tpl b/gnocchi/templates/bin/_ceph-admin-keyring.sh.tpl new file mode 100644 index 0000000000..f19bf03e05 --- /dev/null +++ b/gnocchi/templates/bin/_ceph-admin-keyring.sh.tpl @@ -0,0 +1,29 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +export HOME=/tmp + +cat < /etc/ceph/ceph.client.admin.keyring +[client.admin] +{{- if .Values.conf.ceph.admin_keyring }} + key = {{ .Values.conf.ceph.admin_keyring }} +{{- else }} + key = $(cat /tmp/client-keyring) +{{- end }} +EOF + +exit 0 diff --git a/gnocchi/templates/bin/_ceph-keyring.sh.tpl b/gnocchi/templates/bin/_ceph-keyring.sh.tpl new file mode 100644 index 0000000000..db5f25fe48 --- /dev/null +++ b/gnocchi/templates/bin/_ceph-keyring.sh.tpl @@ -0,0 +1,30 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +export HOME=/tmp + +cat < /etc/ceph/ceph.client.{{ .Values.conf.gnocchi.storage.ceph_username }}.keyring + +[client.{{ .Values.conf.gnocchi.storage.ceph_username }}] +{{- if .Values.conf.gnocchi.storage.provided_keyring }} + key = {{ .Values.conf.gnocchi.storage.provided_keyring }} +{{- else }} + key = $(cat /tmp/client-keyring) +{{- end }} +EOF + +exit 0 diff --git a/gnocchi/templates/bin/_clean-secrets.sh.tpl b/gnocchi/templates/bin/_clean-secrets.sh.tpl new file mode 100644 index 0000000000..31b7177cff --- /dev/null +++ b/gnocchi/templates/bin/_clean-secrets.sh.tpl @@ -0,0 +1,22 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +exec kubectl delete secret \ + --namespace ${NAMESPACE} \ + --ignore-not-found=true \ + ${RBD_POOL_SECRET} diff --git a/gnocchi/templates/bin/_db-init.sh.tpl b/gnocchi/templates/bin/_db-init.sh.tpl new file mode 100644 index 0000000000..b95d4a2148 --- /dev/null +++ b/gnocchi/templates/bin/_db-init.sh.tpl @@ -0,0 +1,89 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +export HOME=/tmp + +pgsql_superuser_cmd () { + DB_COMMAND="$1" + if [[ ! -z $2 ]]; then + EXPORT PGDATABASE=$2 + fi + if [[ ! -z "${ROOT_DB_PASS}" ]]; then + export PGPASSWORD="${ROOT_DB_PASS}" + fi + psql \ + -h ${DB_FQDN} \ + -p ${DB_PORT} \ + -U ${ROOT_DB_USER} \ + --command="${DB_COMMAND}" + unset PGPASSWORD +} + +if [[ ! -v ROOT_DB_CONNECTION ]]; then + echo "environment variable ROOT_DB_CONNECTION not set" + exit 1 +else + echo "Got DB root connection" +fi + +if [[ -v OPENSTACK_CONFIG_FILE ]]; then + if [[ ! -v OPENSTACK_CONFIG_DB_SECTION ]]; then + echo "Environment variable OPENSTACK_CONFIG_DB_SECTION not set" + exit 1 + elif [[ ! -v OPENSTACK_CONFIG_DB_KEY ]]; then + echo "Environment variable OPENSTACK_CONFIG_DB_KEY not set" + exit 1 + fi + + echo "Using ${OPENSTACK_CONFIG_FILE} as db config source" + echo "Trying to load db config from ${OPENSTACK_CONFIG_DB_SECTION}:${OPENSTACK_CONFIG_DB_KEY}" + + DB_CONN=$(awk -v key=$OPENSTACK_CONFIG_DB_KEY "/^\[${OPENSTACK_CONFIG_DB_SECTION}\]/{f=1} f==1&&/^$OPENSTACK_CONFIG_DB_KEY/{print \$3;exit}" "${OPENSTACK_CONFIG_FILE}") + + echo "Found DB connection: $DB_CONN" +elif [[ -v DB_CONNECTION ]]; then + DB_CONN=${DB_CONNECTION} + echo "Got config from DB_CONNECTION env var" +else + echo "Could not get dbconfig" + exit 1 +fi + +ROOT_DB_PROTO="$(echo $ROOT_DB_CONNECTION | grep '//' | sed -e's,^\(.*://\).*,\1,g')" +ROOT_DB_URL="$(echo $ROOT_DB_CONNECTION | sed -e s,$ROOT_DB_PROTO,,g)" +ROOT_DB_USER="$(echo $ROOT_DB_URL | grep @ | cut -d@ -f1 | cut -d: -f1)" +ROOT_DB_PASS="$(echo $ROOT_DB_URL | grep @ | cut -d@ -f1 | cut -d: -f2)" + +DB_FQDN="$(echo $ROOT_DB_URL | sed -e s,$ROOT_DB_USER:$ROOT_DB_PASS@,,g | cut -d/ -f1 | cut -d: -f1)" +DB_PORT="$(echo $ROOT_DB_URL | sed -e s,$ROOT_DB_USER:$ROOT_DB_PASS@,,g | cut -d/ -f1 | cut -d: -f2)" +DB_NAME="$(echo $ROOT_DB_URL | sed -e s,$ROOT_DB_USER:$ROOT_DB_PASS@,,g | cut -d/ -f2 | cut -d? -f1)" + +DB_PROTO="$(echo $DB_CONN | grep '//' | sed -e's,^\(.*://\).*,\1,g')" +DB_URL="$(echo $DB_CONN | sed -e s,$DB_PROTO,,g)" +DB_USER="$( echo $DB_URL | grep @ | cut -d@ -f1 | cut -d: -f1)" +DB_PASS="$( echo $DB_URL | grep @ | cut -d@ -f1 | cut -d: -f2)" + +#create db +pgsql_superuser_cmd "SELECT 1 FROM pg_database WHERE datname = '$DB_NAME'" | grep -q 1 || pgsql_superuser_cmd "CREATE DATABASE $DB_NAME" + +#create db user +pgsql_superuser_cmd "SELECT * FROM pg_roles WHERE rolname = '$DB_USER';" | tail -n +3 | head -n -2 | grep -q 1 || \ + pgsql_superuser_cmd "CREATE ROLE ${DB_USER} LOGIN PASSWORD '$DB_PASS';" && pgsql_superuser_cmd "ALTER USER ${DB_USER} WITH SUPERUSER" + +#give permissions to user +pgsql_superuser_cmd "GRANT ALL PRIVILEGES ON DATABASE $DB_NAME to $DB_USER;" + diff --git a/gnocchi/templates/bin/_db-sync.sh.tpl b/gnocchi/templates/bin/_db-sync.sh.tpl new file mode 100644 index 0000000000..87698f339c --- /dev/null +++ b/gnocchi/templates/bin/_db-sync.sh.tpl @@ -0,0 +1,19 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +exec gnocchi-upgrade diff --git a/gnocchi/templates/bin/_gnocchi-api.sh.tpl b/gnocchi/templates/bin/_gnocchi-api.sh.tpl new file mode 100644 index 0000000000..446fc68b0d --- /dev/null +++ b/gnocchi/templates/bin/_gnocchi-api.sh.tpl @@ -0,0 +1,32 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +COMMAND="${@:-start}" + +function start () { + if [ -f /etc/apache2/envvars ]; then + # Loading Apache2 ENV variables + source /etc/apache2/envvars + fi + exec apache2 -DFOREGROUND +} + +function stop () { + kill -TERM 1 +} + +$COMMAND diff --git a/gnocchi/templates/bin/_gnocchi-metricd.sh.tpl b/gnocchi/templates/bin/_gnocchi-metricd.sh.tpl new file mode 100644 index 0000000000..71c318d155 --- /dev/null +++ b/gnocchi/templates/bin/_gnocchi-metricd.sh.tpl @@ -0,0 +1,19 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -x +exec gnocchi-metricd \ + --config-file /etc/gnocchi/gnocchi.conf diff --git a/gnocchi/templates/bin/_gnocchi-resources-cleaner.sh.tpl b/gnocchi/templates/bin/_gnocchi-resources-cleaner.sh.tpl new file mode 100644 index 0000000000..df03d5ed01 --- /dev/null +++ b/gnocchi/templates/bin/_gnocchi-resources-cleaner.sh.tpl @@ -0,0 +1,20 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +echo "Purging the deleted resources with its associated metrics which have lived more than ${DELETED_RESOURCES_TTL}" +gnocchi resource batch delete "ended_at < '-${DELETED_RESOURCES_TTL}'" + +exit 0 diff --git a/gnocchi/templates/bin/_gnocchi-statsd.sh.tpl b/gnocchi/templates/bin/_gnocchi-statsd.sh.tpl new file mode 100644 index 0000000000..e962e57563 --- /dev/null +++ b/gnocchi/templates/bin/_gnocchi-statsd.sh.tpl @@ -0,0 +1,19 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -x +exec gnocchi-statsd \ + --config-file /etc/gnocchi/gnocchi.conf diff --git a/gnocchi/templates/bin/_gnocchi-test.sh.tpl b/gnocchi/templates/bin/_gnocchi-test.sh.tpl new file mode 100644 index 0000000000..403548540d --- /dev/null +++ b/gnocchi/templates/bin/_gnocchi-test.sh.tpl @@ -0,0 +1,66 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +export HOME=/tmp + +echo "Test: list archive policies" +gnocchi archive-policy list + +echo "Test: create metric" +gnocchi metric create --archive-policy-name low +METRIC_UUID=$(gnocchi metric list -c id -f value | head -1) +sleep 5 + +echo "Test: show metric" +gnocchi metric show ${METRIC_UUID} + +sleep 5 + +echo "Test: add measures" +gnocchi measures add -m 2017-06-27T12:00:00@31 \ + -m 2017-06-27T12:03:27@20 \ + -m 2017-06-27T12:06:51@41 \ + ${METRIC_UUID} + +sleep 15 + +echo "Test: show measures" +gnocchi measures show ${METRIC_UUID} +gnocchi measures show --aggregation min ${METRIC_UUID} + +echo "Test: delete metric" +gnocchi metric delete ${METRIC_UUID} + +RESOURCE_UUID={{ uuidv4 }} + +echo "Test: create resource type" +gnocchi resource-type create --attribute name:string --attribute host:string test + +echo "Test: list resource types" +gnocchi resource-type list + +echo "Test: create resource" +gnocchi resource create --attribute name:test --attribute host:testnode1 --create-metric cpu:medium --create-metric memory:low --type test ${RESOURCE_UUID} + +echo "Test: show resource history" +gnocchi resource history --format json --details ${RESOURCE_UUID} +echo "Test: delete resource" +gnocchi resource delete ${RESOURCE_UUID} +echo "Test: delete resource type" +gnocchi resource-type delete test + +exit 0 diff --git a/gnocchi/templates/bin/_storage-init.sh.tpl b/gnocchi/templates/bin/_storage-init.sh.tpl new file mode 100644 index 0000000000..1710ce04bc --- /dev/null +++ b/gnocchi/templates/bin/_storage-init.sh.tpl @@ -0,0 +1,62 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -x +SECRET=$(mktemp --suffix .yaml) +KEYRING=$(mktemp --suffix .keyring) +function cleanup { + rm -f ${SECRET} ${KEYRING} +} +trap cleanup EXIT + +set -ex +ceph -s +function ensure_pool () { + ceph osd pool stats $1 || ceph osd pool create $1 $2 + local test_version=$(ceph tell osd.* version | egrep -c "nautilus|mimic|luminous" | xargs echo) + if [[ ${test_version} -gt 0 ]]; then + ceph osd pool application enable $1 $3 + fi +} +ensure_pool ${RBD_POOL_NAME} ${RBD_POOL_CHUNK_SIZE} "gnocchi-metrics" + +if USERINFO=$(ceph auth get client.${RBD_POOL_USER}); then + echo "Cephx user client.${RBD_POOL_USER} already exist." + echo "Update its cephx caps" + ceph auth caps client.${RBD_POOL_USER} \ + mon "profile rbd" \ + osd "profile rbd pool=${RBD_POOL_NAME}" \ + mgr "allow r" + ceph auth get client.${RBD_POOL_USER} -o ${KEYRING} +else + ceph auth get-or-create client.${RBD_POOL_USER} \ + mon "profile rbd" \ + osd "profile rbd pool=${RBD_POOL_NAME}" \ + mgr "allow r" \ + -o ${KEYRING} +fi + +ENCODED_KEYRING=$(sed -n 's/^[[:blank:]]*key[[:blank:]]\+=[[:blank:]]\(.*\)/\1/p' ${KEYRING} | base64 -w0) +cat > ${SECRET} < + WSGIDaemonProcess gnocchi processes=1 threads=2 user=gnocchi group=gnocchi display-name=%{GROUP} + WSGIProcessGroup gnocchi + WSGIScriptAlias / "/var/lib/kolla/venv/lib/python2.7/site-packages/gnocchi/rest/app.wsgi" + WSGIApplicationGroup %{GLOBAL} + + ErrorLog /dev/stderr + SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded + CustomLog /dev/stdout combined env=!forwarded + CustomLog /dev/stdout proxy env=forwarded + + + Require all granted + + + ceph: + monitors: [] + admin_keyring: null + override: + append: + enable_paste: True + paste: + pipeline:main: + pipeline: gnocchi+auth + composite:gnocchi+noauth: + use: egg:Paste#urlmap + /: gnocchiversions + /v1: gnocchiv1+noauth + composite:gnocchi+auth: + use: egg:Paste#urlmap + /: gnocchiversions + /v1: gnocchiv1+auth + pipeline:gnocchiv1+noauth: + pipeline: gnocchiv1 + pipeline:gnocchiv1+auth: + pipeline: keystone_authtoken gnocchiv1 + app:gnocchiversions: + paste.app_factory: gnocchi.rest.app:app_factory + root: gnocchi.rest.VersionsController + app:gnocchiv1: + paste.app_factory: gnocchi.rest.app:app_factory + root: gnocchi.rest.V1Controller + filter:keystone_authtoken: + paste.filter_factory: keystonemiddleware.auth_token:filter_factory + oslo_config_project: gnocchi + policy: + admin_or_creator: 'role:admin or project_id:%(created_by_project_id)s' + resource_owner: 'project_id:%(project_id)s' + metric_owner: 'project_id:%(resource.project_id)s' + get status: 'role:admin' + create resource: '' + get resource: 'rule:admin_or_creator or rule:resource_owner' + update resource: 'rule:admin_or_creator' + delete resource: 'rule:admin_or_creator' + delete resources: 'rule:admin_or_creator' + list resource: 'rule:admin_or_creator or rule:resource_owner' + search resource: 'rule:admin_or_creator or rule:resource_owner' + create resource type: 'role:admin' + delete resource type: 'role:admin' + update resource type: 'role:admin' + list resource type: '' + get resource type: '' + get archive policy: '' + list archive policy: '' + create archive policy: 'role:admin' + update archive policy: 'role:admin' + delete archive policy: 'role:admin' + create archive policy rule: 'role:admin' + get archive policy rule: '' + list archive policy rule: '' + delete archive policy rule: 'role:admin' + create metric: '' + delete metric: 'rule:admin_or_creator' + get metric: 'rule:admin_or_creator or rule:metric_owner' + search metric: 'rule:admin_or_creator or rule:metric_owner' + list metric: '' + list all metric: 'role:admin' + get measures: 'rule:admin_or_creator or rule:metric_owner' + post measures: 'rule:admin_or_creator' + gnocchi: + DEFAULT: + debug: false + token: + provider: uuid + api: + auth_mode: keystone + # NOTE(portdirect): the bind port should not be defined, and is manipulated + # via the endpoints section. + port: null + statsd: + # NOTE(portdirect): the bind port should not be defined, and is manipulated + # via the endpoints section. + port: null + metricd: + workers: 1 + database: + max_retries: -1 + storage: + driver: ceph + ceph_pool: gnocchi.metrics + ceph_username: gnocchi + ceph_keyring: /etc/ceph/ceph.client.gnocchi.keyring + ceph_conffile: /etc/ceph/ceph.conf + file_basepath: /var/lib/gnocchi + provided_keyring: null + indexer: + driver: postgresql + keystone_authtoken: + auth_type: password + auth_version: v3 + memcache_security_strategy: ENCRYPT + +ceph_client: + configmap: ceph-etc + user_secret_name: pvc-ceph-client-key + +secrets: + identity: + admin: gnocchi-keystone-admin + gnocchi: gnocchi-keystone-user + oslo_db: + admin: gnocchi-db-admin + gnocchi: gnocchi-db-user + oslo_db_indexer: + admin: gnocchi-db-indexer-admin + gnocchi: gnocchi-db-indexer-user + rbd: gnocchi-rbd-keyring + tls: + metric: + api: + public: gnocchi-tls-public + +bootstrap: + enabled: false + ks_user: gnocchi + script: | + openstack token issue + +# typically overridden by environmental +# values, but should include all endpoints +# required by this chart +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + identity: + name: keystone + auth: + admin: + username: "admin" + user_domain_name: "default" + password: "password" + project_name: "admin" + project_domain_name: "default" + region_name: "RegionOne" + os_auth_type: "password" + os_tenant_name: "admin" + gnocchi: + username: "gnocchi" + role: "admin" + password: "password" + project_name: "service" + region_name: "RegionOne" + os_auth_type: "password" + os_tenant_name: "service" + user_domain_name: service + project_domain_name: service + hosts: + default: keystone + internal: keystone-api + host_fqdn_override: + default: null + path: + default: /v3 + scheme: + default: 'http' + port: + api: + default: 80 + internal: 5000 + metric: + name: gnocchi + hosts: + default: gnocchi-api + public: gnocchi + host_fqdn_override: + default: null + # NOTE: this chart supports TLS for fqdn over-ridden public + # endpoints using the following format: + # public: + # host: null + # tls: + # crt: null + # key: null + path: + default: null + scheme: + default: 'http' + port: + api: + default: 8041 + public: 80 + metric_statsd: + name: gnocchi-statsd + hosts: + default: gnocchi-statsd + host_fqdn_override: + default: null + path: + default: null + scheme: + default: null + port: + statsd: + default: 8125 + oslo_db_postgresql: + auth: + admin: + username: postgres + password: password + gnocchi: + username: gnocchi + password: password + hosts: + default: postgresql + host_fqdn_override: + default: null + path: /gnocchi + scheme: postgresql + port: + postgresql: + default: 5432 + oslo_db: + auth: + admin: + username: root + password: password + gnocchi: + username: gnocchi + password: password + hosts: + default: mariadb + host_fqdn_override: + default: null + path: /gnocchi + scheme: mysql+pymysql + port: + mysql: + default: 3306 + oslo_cache: + auth: + # NOTE(portdirect): this is used to define the value for keystone + # authtoken cache encryption key, if not set it will be populated + # automatically with a random value, but to take advantage of + # this feature all services should be set to use the same key, + # and memcache service. + memcache_secret_key: null + hosts: + default: memcached + host_fqdn_override: + default: null + port: + memcache: + default: 11211 + +manifests: + configmap_bin: true + configmap_etc: true + cron_job_resources_cleaner: true + daemonset_metricd: true + daemonset_statsd: true + deployment_api: true + ingress_api: true + job_bootstrap: true + job_clean: true + job_db_drop: false + job_db_init_indexer: true + job_db_init: true + job_image_repo_sync: true + secret_db_indexer: true + job_db_sync: true + job_ks_endpoints: true + job_ks_service: true + job_ks_user: true + job_storage_init: true + pdb_api: true + pod_gnocchi_test: true + secret_db: true + secret_keystone: true + secret_ingress_tls: true + service_api: true + service_ingress_api: true + service_statsd: true +... diff --git a/grafana/Chart.yaml b/grafana/Chart.yaml new file mode 100644 index 0000000000..10946084db --- /dev/null +++ b/grafana/Chart.yaml @@ -0,0 +1,29 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: v9.2.10 +description: OpenStack-Helm Grafana +name: grafana +version: 2024.2.0 +home: https://grafana.com/ +sources: + - https://github.com/grafana/grafana + - https://opendev.org/openstack/openstack-helm-addons +maintainers: + - name: OpenStack-Helm Authors +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit + version: ">= 0.1.0" +... diff --git a/grafana/templates/bin/_db-session-sync.py.tpl b/grafana/templates/bin/_db-session-sync.py.tpl new file mode 100644 index 0000000000..b145fdfcd3 --- /dev/null +++ b/grafana/templates/bin/_db-session-sync.py.tpl @@ -0,0 +1,70 @@ +#!/usr/bin/env python + +# Creates db and user for an OpenStack Service: +# Set ROOT_DB_CONNECTION and DB_CONNECTION environment variables to contain +# SQLAlchemy strings for the root connection to the database and the one you +# wish the service to use. Alternatively, you can use an ini formatted config +# at the location specified by OPENSTACK_CONFIG_FILE, and extract the string +# from the key OPENSTACK_CONFIG_DB_KEY, in the section specified by +# OPENSTACK_CONFIG_DB_SECTION. + +import os +import sys +import logging +from sqlalchemy import create_engine + +# Create logger, console handler and formatter +logger = logging.getLogger('OpenStack-Helm DB Init') +logger.setLevel(logging.DEBUG) +ch = logging.StreamHandler() +ch.setLevel(logging.DEBUG) +formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(filename)s - %(lineno)d - %(funcName)s - %(message)s') + +# Set the formatter and add the handler +ch.setFormatter(formatter) +logger.addHandler(ch) + +# Get the connection string for the service db +if "DB_CONNECTION" in os.environ: + user_db_conn = os.environ['DB_CONNECTION'] + logger.info('Got config from DB_CONNECTION env var') +else: + logger.critical('Could not get db config, either from config file or env var') + sys.exit(1) + +# User DB engine +try: + user_engine = create_engine(user_db_conn) + # Get our user data out of the user_engine + database = user_engine.url.database + user = user_engine.url.username + password = user_engine.url.password + host = user_engine.url.host + port = user_engine.url.port + logger.info('Got user db config') +except: + logger.critical('Could not get user database config') + raise + +# Test connection +try: + connection = user_engine.connect() + connection.close() + logger.info("Tested connection to DB @ {0}:{1}/{2} as {3}".format( + host, port, database, user)) +except: + logger.critical('Could not connect to database as user') + raise + +# Create Table +try: + user_engine.execute('''CREATE TABLE IF NOT EXISTS `session` ( + `key`CHAR(16) NOT NULL, + `data` BLOB, + `expiry` INT(11) UNSIGNED NOT NULL, + PRIMARY KEY (`key`) + ) ENGINE=MyISAM DEFAULT CHARSET=utf8;''') + logger.info('Created table for session cache') +except: + logger.critical('Could not create table for session cache') + raise diff --git a/grafana/templates/bin/_grafana.sh.tpl b/grafana/templates/bin/_grafana.sh.tpl new file mode 100644 index 0000000000..19e57dcf53 --- /dev/null +++ b/grafana/templates/bin/_grafana.sh.tpl @@ -0,0 +1,127 @@ +#!/bin/bash +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -exo pipefail +COMMAND="${@:-start}" +PORT={{ tuple "grafana" "internal" "grafana" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +PIDFILE=/tmp/pid +DB_HOST={{ tuple "oslo_db" "direct" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }} +DB_PORT={{ tuple "oslo_db" "direct" "mysql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +MYSQL_PARAMS=" \ + --defaults-file=/tmp/my.cnf \ + --host=${DB_HOST} \ + --port=${DB_PORT} +{{- if .Values.manifests.certificates }} + --ssl-verify-server-cert=false \ + --ssl-ca=/etc/mysql/certs/ca.crt \ + --ssl-key=/etc/mysql/certs/tls.key \ + --ssl-cert=/etc/mysql/certs/tls.crt \ +{{- end }} + " + +function start () { + exec /usr/share/grafana/bin/grafana-server -homepath=/usr/share/grafana -config=/etc/grafana/grafana.ini --pidfile="$PIDFILE" +} + +function run_migrator () { + BACKUP_FILE=$(mktemp) + LOG_FILE=$(mktemp) + STOP_FLAG=$(mktemp) + echo "Making sure the database is reachable...." + set +e + until mysql ${MYSQL_PARAMS} grafana -e "select 1;" + do + echo \"Database ${DB_HOST} is not reachable. Sleeping for 10 seconds...\" + sleep 10 + done + set -e + echo "Preparing initial database backup..." + mysqldump ${MYSQL_PARAMS} --add-drop-table --quote-names grafana > "${BACKUP_FILE}" + echo "Backup SQL file ${BACKUP_FILE}" + ls -lh "${BACKUP_FILE}" + { + # this is the background process that re-starts grafana-server + # in prder to process grafana database migration + set +e + while true + do + start 2>&1 | tee "$LOG_FILE" + sleep 10 + echo "Restarting the grafana-server..." + stop + echo "Emptying log file..." + echo > "$LOG_FILE" + while [ -f ${STOP_FLAG} ] + do + echo "Lock file still exists - ${STOP_FLAG}..." + ls -la ${STOP_FLAG} + echo "Waiting for lock file to get removed..." + sleep 5 + done + echo "Lock file is removed, proceeding with grafana re-start.." + done + set -e + } & + until cat "${LOG_FILE}" | grep -E "migrations completed" + do + echo "The migrations are not completed yet..." + if cat "${LOG_FILE}" | grep -E "migration failed" + then + echo "Locking server restart by placing a flag file ${STOP_FLAG} .." + touch "${STOP_FLAG}" + echo "Migration failure has been detected. Stopping the grafana-server..." + set +e + stop + set -e + echo "Making sure the database is reachable...." + set +e + until mysql ${MYSQL_PARAMS} grafana -e "select 1;" + do + echo \"Database ${DB_HOST} is not reachable. Sleeping for 10 seconds...\" + sleep 10 + done + set -e + echo "Cleaning the database..." + TABLES=$( + mysql ${MYSQL_PARAMS} grafana -e "show tables\G;" | grep Tables | cut -d " " -f 2 + ) + for TABLE in ${TABLES} + do + echo ${TABLE} + mysql ${MYSQL_PARAMS} grafana -e "drop table ${TABLE};" + done + echo "Restoring the database backup..." + mysql ${MYSQL_PARAMS} grafana < "${BACKUP_FILE}" + echo "Removing lock file ${STOP_FLAG} ..." + rm -f "${STOP_FLAG}" + echo "${STOP_FLAG} has been removed" + fi + sleep 10 + done + stop + rm -f "${BACKUP_FILE}" +} + +function stop () { + if [ -f "$PIDFILE" ]; then + echo -e "Found pidfile, killing running grafana-server" + kill -9 `cat $PIDFILE` + rm $PIDFILE + else + kill -TERM 1 + fi +} + +$COMMAND diff --git a/grafana/templates/bin/_selenium-tests.py.tpl b/grafana/templates/bin/_selenium-tests.py.tpl new file mode 100644 index 0000000000..214a09a662 --- /dev/null +++ b/grafana/templates/bin/_selenium-tests.py.tpl @@ -0,0 +1,98 @@ +#!/usr/bin/env python3 + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +import logging +import os +import sys +from selenium import webdriver +from selenium.webdriver.common.by import By +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support import expected_conditions as EC +from selenium.webdriver.chrome.options import Options +{{- if .Values.selenium_v4 }} +from selenium.webdriver.chrome.service import Service +{{- end }} +from selenium.common.exceptions import TimeoutException +from selenium.common.exceptions import NoSuchElementException + +# Create logger, console handler and formatter +logger = logging.getLogger('Grafana Selenium Tests') +logger.setLevel(logging.DEBUG) +ch = logging.StreamHandler() +ch.setLevel(logging.DEBUG) +formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s' +) + +# Set the formatter and add the handler +ch.setFormatter(formatter) +logger.addHandler(ch) + +def get_variable(env_var): + if env_var in os.environ: + logger.info('Found "{}"'.format(env_var)) + return os.environ[env_var] + else: + logger.critical('Variable "{}" is not defined!'.format(env_var)) + sys.exit(1) + +username = get_variable('GRAFANA_USER') +password = get_variable('GRAFANA_PASSWORD') +grafana_uri = get_variable('GRAFANA_URI') + +chrome_driver = '/etc/selenium/chromedriver' +options = Options() +options.add_argument('--headless=new') +options.add_argument('--no-sandbox') +options.add_argument('--window-size=1920x1080') + +{{- if .Values.selenium_v4 }} +service = Service(executable_path=chrome_driver) +browser = webdriver.Chrome(service=service, options=options) +{{- else }} +browser = webdriver.Chrome(chrome_driver, chrome_options=options) +{{- end }} + +logger.info("Attempting to open Grafana dashboard") +try: + browser.get(grafana_uri) + el = WebDriverWait(browser, 15).until( + EC.title_contains('Grafana') + ) + logger.info('Connected to Grafana') +except TimeoutException: + logger.critical('Timed out waiting for Grafana') + browser.quit() + sys.exit(1) + +logger.info("Attempting to log into Grafana dashboard") +try: +{{- if .Values.selenium_v4 }} + browser.find_element(By.NAME, 'user').send_keys(username) + browser.find_element(By.NAME, 'password').send_keys(password) + browser.find_element(By.CSS_SELECTOR, '[type="submit"]').click() +{{- else }} + browser.find_element_by_name('user').send_keys(username) + browser.find_element_by_name('password').send_keys(password) + browser.find_element_by_css_selector('[type="submit"]').click() +{{- end }} + logger.info("Successfully logged in to Grafana") +except NoSuchElementException: + logger.error("Failed to log in to Grafana") + browser.quit() + sys.exit(1) + +browser.quit() diff --git a/grafana/templates/bin/_set-admin-password.sh.tpl b/grafana/templates/bin/_set-admin-password.sh.tpl new file mode 100644 index 0000000000..0feecfd8fa --- /dev/null +++ b/grafana/templates/bin/_set-admin-password.sh.tpl @@ -0,0 +1,24 @@ +#!/bin/bash +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +echo "Attempting to update Grafana admin user password" +grafana-cli --homepath "/usr/share/grafana" --config /etc/grafana/grafana.ini admin reset-admin-password ${GF_SECURITY_ADMIN_PASSWORD} + +if [ "$?" == 1 ]; then + echo "The Grafana admin user does not exist yet, so no need to update password" + exit 0; +else + exit 0; +fi diff --git a/grafana/templates/configmap-bin.yaml b/grafana/templates/configmap-bin.yaml new file mode 100644 index 0000000000..b415258e68 --- /dev/null +++ b/grafana/templates/configmap-bin.yaml @@ -0,0 +1,35 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_bin }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-bin +data: + db-init.py: | +{{- include "helm-toolkit.scripts.db_init" . | indent 4 }} + db-session-sync.py: | +{{ tuple "bin/_db-session-sync.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + image-repo-sync.sh: | +{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} + grafana.sh: | +{{ tuple "bin/_grafana.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + selenium-tests.py: | +{{ tuple "bin/_selenium-tests.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + set-admin-password.sh: | +{{ tuple "bin/_set-admin-password.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} diff --git a/grafana/templates/configmap-dashboards.yaml b/grafana/templates/configmap-dashboards.yaml new file mode 100644 index 0000000000..633295fcbf --- /dev/null +++ b/grafana/templates/configmap-dashboards.yaml @@ -0,0 +1,27 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_dashboards }} +{{ range $group, $dashboards := .Values.conf.dashboards }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: grafana-dashboards-{{$group}} +data: +{{ range $key, $value := $dashboards }} + {{$key}}.json: {{ $value | toJson }} +{{ end }} +{{ end }} +{{- end }} diff --git a/grafana/templates/configmap-etc.yaml b/grafana/templates/configmap-etc.yaml new file mode 100644 index 0000000000..4ce4f34696 --- /dev/null +++ b/grafana/templates/configmap-etc.yaml @@ -0,0 +1,49 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_etc }} +{{- $envAll := . }} + +{{- if and (empty .Values.conf.grafana.database.url) (not (eq .Values.conf.grafana.database.type "sqlite3") ) -}} + +{{- $url := tuple "oslo_db" "internal" "user" "mysql" . | include "helm-toolkit.endpoints.authenticated_endpoint_uri_lookup" | replace "mysql+pymysql://" "mysql://" -}} +{{- if .Values.manifests.certificates -}} +{{- $_ := (printf "%s?charset=utf8" $url ) | set .Values.conf.grafana.database "url" -}} +{{- $_ := tuple "oslo_db" "internal" . | include "helm-toolkit.endpoints.endpoint_host_lookup" | set .Values.conf.grafana.database "server_cert_name" -}} +{{- else -}} +{{- $_ := set .Values.conf.grafana.database "url" $url -}} +{{- end -}} +{{- end -}} + +{{- if empty .Values.conf.grafana.session.provider_config -}} +{{- $user := .Values.endpoints.oslo_db_session.auth.user.username }} +{{- $pass := .Values.endpoints.oslo_db_session.auth.user.password }} +{{- $host_port := tuple "oslo_db_session" "internal" "mysql" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} +{{- $path := .Values.endpoints.oslo_db_session.path }} +{{- $_ := printf "%s:%s%s(%s)%s" $user $pass "@tcp" $host_port $path | set .Values.conf.grafana.session "provider_config" }} +{{- end -}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: grafana-etc +type: Opaque +data: + dashboards.yaml: {{ toYaml .Values.conf.provisioning.dashboards | b64enc }} + grafana.ini: {{ include "helm-toolkit.utils.to_ini" .Values.conf.grafana | b64enc }} +{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.provisioning.datasources.template "key" "datasources.yaml" "format" "Secret") | indent 2 }} +{{ if not (empty .Values.conf.ldap) }} +{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.ldap.template "key" "ldap.toml" "format" "Secret") | indent 2 }} +{{ end }} +{{- end }} diff --git a/grafana/templates/deployment.yaml b/grafana/templates/deployment.yaml new file mode 100644 index 0000000000..2bb980d43e --- /dev/null +++ b/grafana/templates/deployment.yaml @@ -0,0 +1,199 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.deployment }} +{{- $envAll := . }} + +{{- $mounts_grafana := .Values.pod.mounts.grafana.grafana }} + +{{- $serviceAccountName := "grafana" }} +{{ tuple $envAll "grafana" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: grafana + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "grafana" "dashboard" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + replicas: {{ .Values.pod.replicas.grafana }} + selector: + matchLabels: +{{ tuple $envAll "grafana" "dashboard" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} +{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "grafana" "dashboard" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} +{{ dict "envAll" $envAll "podName" "grafana" "containerNames" (list "grafana" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "dashboard" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + affinity: +{{ tuple $envAll "grafana" "dashboard" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} + nodeSelector: + {{ .Values.labels.grafana.node_selector_key }}: {{ .Values.labels.grafana.node_selector_value | quote }} + initContainers: +{{ tuple $envAll "grafana" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} +{{- if and .Values.conf.grafana.image_rendering_sidecar.enabled .Values.conf.grafana.image_rendering_sidecar.k8s_sidecar_feature_enabled }} + - name: grafana-image-renderer +{{ tuple $envAll "grafana_image_renderer" | include "helm-toolkit.snippets.image" | indent 10 }} + restartPolicy: Always + ports: + - containerPort: {{ tuple "grafana" "image_rendering" "grafana" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + readinessProbe: + tcpSocket: + port: {{ tuple "grafana" "image_rendering" "grafana" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + initialDelaySeconds: 15 + periodSeconds: 10 +{{- end }} + containers: +{{- if and .Values.conf.grafana.image_rendering_sidecar.enabled (not .Values.conf.grafana.image_rendering_sidecar.k8s_sidecar_feature_enabled) }} + - name: grafana-image-renderer +{{ tuple $envAll "grafana_image_renderer" | include "helm-toolkit.snippets.image" | indent 10 }} + ports: + - containerPort: {{ tuple "grafana" "image_rendering" "grafana" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + readinessProbe: + tcpSocket: + port: {{ tuple "grafana" "image_rendering" "grafana" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + initialDelaySeconds: 15 + periodSeconds: 10 +{{- end }} + - name: grafana +{{ tuple $envAll "grafana" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.grafana | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "dashboard" "container" "grafana" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/grafana.sh + - start + ports: + - name: dashboard + containerPort: {{ tuple "grafana" "internal" "grafana" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + readinessProbe: + httpGet: + path: /login + port: {{ tuple "grafana" "internal" "grafana" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + initialDelaySeconds: 30 + timeoutSeconds: 30 + env: + - name: GF_SECURITY_ADMIN_USER + valueFrom: + secretKeyRef: + name: grafana-admin-creds + key: GRAFANA_ADMIN_USERNAME + - name: GF_SECURITY_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: grafana-admin-creds + key: GRAFANA_ADMIN_PASSWORD + - name: PROMETHEUS_URL + value: {{ tuple "monitoring" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" }} +{{- if .Values.manifests.certificates }} + - name: CACERT + valueFrom: + secretKeyRef: + key: ca.crt + name: prometheus-tls-api +{{- end }} +{{- if .Values.conf.grafana.image_rendering_sidecar.enabled }} + - name: GF_RENDERING_SERVER_URL + value: "http://localhost:{{ tuple "grafana" "image_rendering" "grafana" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/render" + - name: GF_RENDERING_CALLBACK_URL + value: "http://localhost:{{ tuple "grafana" "internal" "grafana" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}" +{{- end }} +{{- if .Values.pod.env.grafana }} +{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.grafana | indent 12 }} +{{- end }} + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: pod-etc-grafana + mountPath: /etc/grafana + - name: pod-screenshots-grafana + mountPath: /var/lib/grafana/png + - name: pod-dashboards-grafana + mountPath: /etc/grafana/dashboards + - name: pod-provisioning-grafana + mountPath: {{ .Values.conf.grafana.paths.provisioning }} + - name: pod-alerting-grafana + mountPath: {{ .Values.conf.grafana.paths.alerting }} + - name: pod-csv-grafana + mountPath: {{ .Values.conf.grafana.paths.csv }} + - name: grafana-bin + mountPath: /tmp/grafana.sh + subPath: grafana.sh + readOnly: true + - name: grafana-etc + mountPath: {{ .Values.conf.grafana.paths.provisioning }}/dashboards/dashboards.yaml + subPath: dashboards.yaml + - name: grafana-etc + mountPath: {{ .Values.conf.grafana.paths.provisioning }}/datasources/datasources.yaml + subPath: datasources.yaml + - name: grafana-etc + mountPath: /etc/grafana/grafana.ini + subPath: grafana.ini + - name: grafana-etc + mountPath: /etc/grafana/ldap.toml + subPath: ldap.toml + - name: data + mountPath: /var/lib/grafana/data + {{- range $group, $dashboards := .Values.conf.dashboards }} + {{- range $key, $value := $dashboards }} + - name: grafana-dashboards-{{$group}} + mountPath: /etc/grafana/dashboards/{{$key}}.json + subPath: {{$key}}.json + {{- end }} + {{- end }} + +{{- dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.endpoints.oslo_db.auth.admin.secret.tls.internal "path" "/etc/mysql/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }} +{{ if $mounts_grafana.volumeMounts }}{{ toYaml $mounts_grafana.volumeMounts | indent 12 }}{{ end }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: pod-etc-grafana + emptyDir: {} + - name: pod-screenshots-grafana + emptyDir: {} + - name: pod-dashboards-grafana + emptyDir: {} + - name: pod-provisioning-grafana + emptyDir: {} + - name: pod-alerting-grafana + emptyDir: {} + - name: pod-csv-grafana + emptyDir: {} + - name: grafana-bin + configMap: + name: grafana-bin + defaultMode: 0555 + - name: grafana-etc + secret: + secretName: grafana-etc + defaultMode: 0444 + {{- range $group, $dashboards := .Values.conf.dashboards }} + - name: grafana-dashboards-{{$group}} + configMap: + name: grafana-dashboards-{{$group}} + defaultMode: 0555 + {{- end }} + - name: data + emptyDir: {} +{{- dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.endpoints.oslo_db.auth.admin.secret.tls.internal | include "helm-toolkit.snippets.tls_volume" | indent 8 }} +{{ if $mounts_grafana.volumes }}{{ toYaml $mounts_grafana.volumes | indent 8 }}{{ end }} +{{- end }} diff --git a/grafana/templates/ingress-grafana.yaml b/grafana/templates/ingress-grafana.yaml new file mode 100644 index 0000000000..4e27124181 --- /dev/null +++ b/grafana/templates/ingress-grafana.yaml @@ -0,0 +1,22 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.ingress .Values.network.grafana.ingress.public }} +{{- $envAll := . -}} +{{- $ingressOpts := dict "envAll" $envAll "backendService" "grafana" "backendServiceType" "grafana" "backendPort" "dashboard" -}} +{{- if .Values.manifests.certificates -}} +{{- $_ := set $ingressOpts "certIssuer" .Values.endpoints.grafana.host_fqdn_override.default.tls.issuerRef.name -}} +{{- end -}} +{{ $ingressOpts | include "helm-toolkit.manifests.ingress" }} +{{- end }} diff --git a/grafana/templates/job-db-init-session.yaml b/grafana/templates/job-db-init-session.yaml new file mode 100644 index 0000000000..a23fbaba6b --- /dev/null +++ b/grafana/templates/job-db-init-session.yaml @@ -0,0 +1,84 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.job_db_init_session }} +{{- $envAll := . }} + +{{- $serviceAccountName := "grafana-db-init-session" }} +{{ tuple $envAll "db_init_session" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: grafana-db-init-session + labels: +{{ tuple $envAll "grafana" "db-init" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +spec: + template: + metadata: + labels: +{{ tuple $envAll "grafana" "db-init" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} +{{ dict "envAll" $envAll "podName" "grafana-db-init-session" "containerNames" (list "grafana-db-init-session" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "db_init" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value | quote }} + initContainers: +{{ tuple $envAll "db_init_session" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: grafana-db-init-session +{{ tuple $envAll "db_init" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.db_init_session | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "db_init" "container" "grafana_db_init_session" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + env: + - name: ROOT_DB_CONNECTION + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.oslo_db_session.admin }} + key: DB_CONNECTION + - name: DB_CONNECTION + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.oslo_db_session.user }} + key: DB_CONNECTION +{{- if $envAll.Values.manifests.certificates }} + - name: MARIADB_X509 + value: "REQUIRE X509" +{{- end }} + command: + - /tmp/db-init.py + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: grafana-bin + mountPath: /tmp/db-init.py + subPath: db-init.py + readOnly: true +{{- dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.endpoints.oslo_db_session.auth.admin.secret.tls.internal "path" "/etc/mysql/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: grafana-bin + configMap: + name: grafana-bin + defaultMode: 0555 +{{- dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.endpoints.oslo_db_session.auth.admin.secret.tls.internal | include "helm-toolkit.snippets.tls_volume" | indent 8 }} +{{- end }} diff --git a/grafana/templates/job-db-init.yaml b/grafana/templates/job-db-init.yaml new file mode 100644 index 0000000000..c69ea7277c --- /dev/null +++ b/grafana/templates/job-db-init.yaml @@ -0,0 +1,84 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.job_db_init }} +{{- $envAll := . }} + +{{- $serviceAccountName := "grafana-db-init" }} +{{ tuple $envAll "db_init" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: grafana-db-init + labels: +{{ tuple $envAll "grafana" "db-init" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +spec: + template: + metadata: + labels: +{{ tuple $envAll "grafana" "db-init" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} +{{ dict "envAll" $envAll "podName" "grafana-db-init" "containerNames" (list "grafana-db-init" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "db_init" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value | quote }} + initContainers: +{{ tuple $envAll "db_init" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: grafana-db-init +{{ tuple $envAll "db_init" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.db_init | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "db_init" "container" "grafana_db_init" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + env: + - name: ROOT_DB_CONNECTION + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.oslo_db.admin }} + key: DB_CONNECTION + - name: DB_CONNECTION + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.oslo_db.user }} + key: DB_CONNECTION +{{- if $envAll.Values.manifests.certificates }} + - name: MARIADB_X509 + value: "REQUIRE X509" +{{- end }} + command: + - /tmp/db-init.py + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: grafana-bin + mountPath: /tmp/db-init.py + subPath: db-init.py + readOnly: true +{{- dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.endpoints.oslo_db.auth.admin.secret.tls.internal "path" "/etc/mysql/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: grafana-bin + configMap: + name: grafana-bin + defaultMode: 0555 +{{- dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.endpoints.oslo_db.auth.admin.secret.tls.internal | include "helm-toolkit.snippets.tls_volume" | indent 8 }} +{{- end }} diff --git a/grafana/templates/job-db-session-sync.yaml b/grafana/templates/job-db-session-sync.yaml new file mode 100644 index 0000000000..cc2c1d7ef2 --- /dev/null +++ b/grafana/templates/job-db-session-sync.yaml @@ -0,0 +1,79 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.job_db_session_sync }} +{{- $envAll := . }} + +{{- $serviceAccountName := "grafana-db-session-sync" }} +{{ tuple $envAll "db_session_sync" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: grafana-db-session-sync + labels: +{{ tuple $envAll "grafana" "db-session-sync" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +spec: + template: + metadata: + labels: +{{ tuple $envAll "grafana" "db-session-sync" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} +{{ dict "envAll" $envAll "podName" "grafana-db-session-sync" "containerNames" (list "grafana-db-session-sync" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "db_session_sync" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value | quote }} + initContainers: +{{ tuple $envAll "db_session_sync" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: grafana-db-session-sync +{{ tuple $envAll "grafana_db_session_sync" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.db_session_sync | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "db_session_sync" "container" "grafana_db_session_sync" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + env: + - name: DB_CONNECTION + valueFrom: + secretKeyRef: + name: {{ .Values.secrets.oslo_db_session.user }} + key: DB_CONNECTION +{{- if $envAll.Values.manifests.certificates }} + - name: MARIADB_X509 + value: "REQUIRE X509" +{{- end }} + command: + - /tmp/db-session-sync.py + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: grafana-bin + mountPath: /tmp/db-session-sync.py + subPath: db-session-sync.py + readOnly: true +{{- dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.endpoints.oslo_db_session.auth.admin.secret.tls.internal "path" "/etc/mysql/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: grafana-bin + configMap: + name: grafana-bin + defaultMode: 0555 +{{- dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.endpoints.oslo_db_session.auth.admin.secret.tls.internal | include "helm-toolkit.snippets.tls_volume" | indent 8 }} +{{- end }} diff --git a/grafana/templates/job-image-repo-sync.yaml b/grafana/templates/job-image-repo-sync.yaml new file mode 100644 index 0000000000..8963d0f9bd --- /dev/null +++ b/grafana/templates/job-image-repo-sync.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "grafana" -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/grafana/templates/job-run-migrator.yaml b/grafana/templates/job-run-migrator.yaml new file mode 100644 index 0000000000..d87e925016 --- /dev/null +++ b/grafana/templates/job-run-migrator.yaml @@ -0,0 +1,210 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.job_run_migrator }} +{{- $envAll := . }} + +{{- $mounts_grafana := .Values.pod.mounts.grafana.grafana }} + +{{- $serviceAccountName := "grafana-run-migrator" }} +{{ tuple $envAll "run_migrator" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: prepare-grafana-migrator + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +data: + prepare-grafana-migrator.sh: | + #!/bin/bash + set -xe + cp -av /usr/share/grafana/* /usr/share/grafana-prepare/ + exit 0 +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: grafana-run-migrator + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "grafana" "run-migrator" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + template: + metadata: + labels: +{{ tuple $envAll "grafana" "run-migrator" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} +{{ dict "envAll" $envAll "podName" "grafana-run-migrator" "containerNames" (list "prepare-grafana-migrator" "grafana-run-migrator" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "run_migrator" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value | quote }} + initContainers: +{{ tuple $envAll "run_migrator" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + - name: prepare-grafana-migrator +{{ tuple $envAll "grafana" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ dict "envAll" $envAll "application" "run_migrator" "container" "prepare_grafana_migrator" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/prepare-grafana-migrator.sh + resources: {} + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: grafana-binary-image + mountPath: /usr/share/grafana-prepare + - name: prepare-grafana-migrator + mountPath: /tmp/prepare-grafana-migrator.sh + readOnly: true + subPath: prepare-grafana-migrator.sh + containers: + - name: grafana-run-migrator +{{ tuple $envAll "mariadb" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.run_migrator | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "run_migrator" "container" "grafana_run_migrator" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/grafana.sh + - run_migrator + ports: + - name: dashboard + containerPort: {{ tuple "grafana" "internal" "grafana" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + # readinessProbe: + # httpGet: + # path: /login + # port: {{ tuple "grafana" "internal" "grafana" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + # initialDelaySeconds: 30 + # timeoutSeconds: 30 + env: + - name: GF_SECURITY_ADMIN_USER + valueFrom: + secretKeyRef: + name: grafana-admin-creds + key: GRAFANA_ADMIN_USERNAME + - name: GF_SECURITY_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: grafana-admin-creds + key: GRAFANA_ADMIN_PASSWORD + - name: PROMETHEUS_URL + value: {{ tuple "monitoring" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" }} +{{- if .Values.manifests.certificates }} + - name: CACERT + valueFrom: + secretKeyRef: + key: ca.crt + name: prometheus-tls-api +{{- end }} +{{- if .Values.pod.env.grafana }} +{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.grafana | indent 12 }} +{{- end }} +{{- if .Values.pod.env.grafana_run_migrator }} +{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.grafana_run_migrator | indent 12 }} +{{- end }} + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: pod-etc-grafana + mountPath: /etc/grafana + - name: pod-screenshots-grafana + mountPath: /var/lib/grafana/png + - name: pod-dashboards-grafana + mountPath: /etc/grafana/dashboards + - name: pod-provisioning-grafana + mountPath: {{ .Values.conf.grafana.paths.provisioning }} + - name: pod-alerting-grafana + mountPath: {{ .Values.conf.grafana.paths.alerting }} + - name: pod-csv-grafana + mountPath: {{ .Values.conf.grafana.paths.csv }} + - name: grafana-binary-image + mountPath: /usr/share/grafana + - name: grafana-bin + mountPath: /tmp/grafana.sh + subPath: grafana.sh + readOnly: true + - name: grafana-etc + mountPath: {{ .Values.conf.grafana.paths.provisioning }}/dashboards/dashboards.yaml + subPath: dashboards.yaml + - name: grafana-etc + mountPath: {{ .Values.conf.grafana.paths.provisioning }}/datasources/datasources.yaml + subPath: datasources.yaml + - name: grafana-etc + mountPath: /etc/grafana/grafana.ini + subPath: grafana.ini + - name: grafana-etc + mountPath: /etc/grafana/ldap.toml + subPath: ldap.toml + - name: grafana-db + mountPath: /tmp/my.cnf + subPath: my.cnf + - name: data + mountPath: /var/lib/grafana/data + {{- range $group, $dashboards := .Values.conf.dashboards }} + {{- range $key, $value := $dashboards }} + - name: grafana-dashboards-{{$group}} + mountPath: /etc/grafana/dashboards/{{$key}}.json + subPath: {{$key}}.json + {{- end }} + {{- end }} + +{{- dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.endpoints.oslo_db.auth.admin.secret.tls.internal "path" "/etc/mysql/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }} +{{ if $mounts_grafana.volumeMounts }}{{ toYaml $mounts_grafana.volumeMounts | indent 12 }}{{ end }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: pod-etc-grafana + emptyDir: {} + - name: pod-screenshots-grafana + emptyDir: {} + - name: pod-dashboards-grafana + emptyDir: {} + - name: pod-provisioning-grafana + emptyDir: {} + - name: pod-alerting-grafana + emptyDir: {} + - name: pod-csv-grafana + emptyDir: {} + - name: grafana-binary-image + emptyDir: {} + - name: grafana-bin + configMap: + name: grafana-bin + defaultMode: 0555 + - name: grafana-etc + secret: + secretName: grafana-etc + defaultMode: 0444 + - name: grafana-db + secret: + secretName: grafana-db + defaultMode: 0444 + {{- range $group, $dashboards := .Values.conf.dashboards }} + - name: grafana-dashboards-{{$group}} + configMap: + name: grafana-dashboards-{{$group}} + defaultMode: 0555 + {{- end }} + - name: data + emptyDir: {} + - name: prepare-grafana-migrator + configMap: + defaultMode: 0555 + name: prepare-grafana-migrator +{{- dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.endpoints.oslo_db.auth.admin.secret.tls.internal | include "helm-toolkit.snippets.tls_volume" | indent 8 }} +{{ if $mounts_grafana.volumes }}{{ toYaml $mounts_grafana.volumes | indent 8 }}{{ end }} +{{- end }} diff --git a/grafana/templates/job-set-admin-user.yaml b/grafana/templates/job-set-admin-user.yaml new file mode 100644 index 0000000000..388ab830b1 --- /dev/null +++ b/grafana/templates/job-set-admin-user.yaml @@ -0,0 +1,87 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.job_set_admin_user }} +{{- $envAll := . }} + +{{- $serviceAccountName := "grafana-set-admin-user" }} +{{ tuple $envAll "set_admin_user" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: grafana-set-admin-user + labels: +{{ tuple $envAll "grafana" "set-admin-user" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +spec: + template: + metadata: + labels: +{{ tuple $envAll "grafana" "set-admin-user" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} +{{ dict "envAll" $envAll "podName" "grafana-set-admin-user" "containerNames" (list "grafana-set-admin-password" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "set_admin_user" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value | quote }} + initContainers: +{{ tuple $envAll "set_admin_user" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: grafana-set-admin-password +{{ tuple $envAll "grafana" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.set_admin_user | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "set_admin_user" "container" "grafana_set_admin_password" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/set-admin-password.sh + env: + - name: GF_SECURITY_ADMIN_USER + valueFrom: + secretKeyRef: + name: grafana-admin-creds + key: GRAFANA_ADMIN_USERNAME + - name: GF_SECURITY_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: grafana-admin-creds + key: GRAFANA_ADMIN_PASSWORD + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: grafana-etc + mountPath: /etc/grafana/grafana.ini + subPath: grafana.ini + - name: grafana-bin + mountPath: /tmp/set-admin-password.sh + subPath: set-admin-password.sh + readOnly: true + volumes: + - name: pod-tmp + emptyDir: {} + - name: pod-etc-grafana + emptyDir: {} + - name: grafana-bin + configMap: + name: grafana-bin + defaultMode: 0555 + - name: grafana-etc + secret: + secretName: grafana-etc + defaultMode: 0444 +{{- end }} diff --git a/grafana/templates/network_policy.yaml b/grafana/templates/network_policy.yaml new file mode 100644 index 0000000000..d178c8a514 --- /dev/null +++ b/grafana/templates/network_policy.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.network_policy -}} +{{- $netpol_opts := dict "envAll" . "name" "application" "label" "grafana" -}} +{{ $netpol_opts | include "helm-toolkit.manifests.kubernetes_network_policy" }} +{{- end -}} diff --git a/grafana/templates/pod-helm-tests.yaml b/grafana/templates/pod-helm-tests.yaml new file mode 100644 index 0000000000..15430798a4 --- /dev/null +++ b/grafana/templates/pod-helm-tests.yaml @@ -0,0 +1,80 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.helm_tests }} +{{- $dashboardCount := len .Values.conf.dashboards }} +{{- $envAll := . }} + +{{- $serviceAccountName := print .Release.Name "-test" }} +{{ tuple $envAll "tests" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: v1 +kind: Pod +metadata: + name: "{{.Release.Name}}-test" + labels: +{{ tuple $envAll "grafana" "test" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: + "helm.sh/hook": test-success + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +{{ dict "envAll" $envAll "podName" "grafana-test" "containerNames" (list "init" "grafana-selenium-tests") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 4 }} +spec: +{{ dict "envAll" $envAll "application" "test" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 2 }} + serviceAccountName: {{ $serviceAccountName }} + nodeSelector: + {{ .Values.labels.test.node_selector_key }}: {{ .Values.labels.test.node_selector_value }} + restartPolicy: Never + initContainers: +{{ tuple $envAll "tests" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 4 }} + containers: + - name: grafana-selenium-tests +{{ tuple $envAll "selenium_tests" | include "helm-toolkit.snippets.image" | indent 6 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.tests | include "helm-toolkit.snippets.kubernetes_resources" | indent 6 }} +{{ dict "envAll" $envAll "application" "test" "container" "helm_tests" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 6 }} + command: + - /tmp/selenium-tests.py + env: + - name: GRAFANA_USER + valueFrom: + secretKeyRef: + name: grafana-admin-creds + key: GRAFANA_ADMIN_USERNAME + - name: GRAFANA_PASSWORD + valueFrom: + secretKeyRef: + name: grafana-admin-creds + key: GRAFANA_ADMIN_PASSWORD + - name: GRAFANA_URI + value: {{ tuple "grafana" "internal" "grafana" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" }} + - name: CHROME_CONFIG_HOME + value: /tmp/google-chrome + - name: XDG_CONFIG_HOME + value: /tmp/google-chrome + - name: XDG_CACHE_HOME + value: /tmp/google-chrome + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: grafana-bin + mountPath: /tmp/selenium-tests.py + subPath: selenium-tests.py + readOnly: true + volumes: + - name: pod-tmp + emptyDir: {} + - name: grafana-bin + configMap: + name: grafana-bin + defaultMode: 0555 +{{- end }} diff --git a/grafana/templates/secret-admin-creds.yaml b/grafana/templates/secret-admin-creds.yaml new file mode 100644 index 0000000000..d80a7ad0bd --- /dev/null +++ b/grafana/templates/secret-admin-creds.yaml @@ -0,0 +1,26 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_admin_creds }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: grafana-admin-creds +type: Opaque +data: + GRAFANA_ADMIN_PASSWORD: {{ .Values.endpoints.grafana.auth.admin.password | b64enc }} + GRAFANA_ADMIN_USERNAME: {{ .Values.endpoints.grafana.auth.admin.username | b64enc }} +{{- end }} diff --git a/grafana/templates/secret-db-session.yaml b/grafana/templates/secret-db-session.yaml new file mode 100644 index 0000000000..82c32ca615 --- /dev/null +++ b/grafana/templates/secret-db-session.yaml @@ -0,0 +1,33 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_db_session }} +{{- $envAll := . }} +{{- range $key1, $userClass := tuple "admin" "user" }} +{{- $secretName := index $envAll.Values.secrets.oslo_db_session $userClass }} +{{- $connection := tuple "oslo_db_session" "internal" $userClass "mysql" $envAll | include "helm-toolkit.endpoints.authenticated_endpoint_uri_lookup" }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} +type: Opaque +data: +{{- if $envAll.Values.manifests.certificates }} + DB_CONNECTION: {{ (printf "%s?charset=utf8&ssl_ca=/etc/mysql/certs/ca.crt&ssl_key=/etc/mysql/certs/tls.key&ssl_cert=/etc/mysql/certs/tls.crt&ssl_verify_cert" $connection ) | b64enc -}} +{{- else }} + DB_CONNECTION: {{ $connection | b64enc -}} +{{- end }} +{{- end }} +{{- end }} diff --git a/grafana/templates/secret-db.yaml b/grafana/templates/secret-db.yaml new file mode 100644 index 0000000000..5d50ec8c3b --- /dev/null +++ b/grafana/templates/secret-db.yaml @@ -0,0 +1,41 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_db }} +{{- $envAll := . }} +{{- range $key1, $userClass := tuple "admin" "user" }} +{{- $secretName := index $envAll.Values.secrets.oslo_db $userClass }} +{{- $connection := tuple "oslo_db" "internal" $userClass "mysql" $envAll | include "helm-toolkit.endpoints.authenticated_endpoint_uri_lookup" }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} +type: Opaque +data: +{{- if $envAll.Values.manifests.certificates }} + DB_CONNECTION: {{ (printf "%s?charset=utf8&ssl_ca=/etc/mysql/certs/ca.crt&ssl_key=/etc/mysql/certs/tls.key&ssl_cert=/etc/mysql/certs/tls.crt&ssl_verify_cert" $connection ) | b64enc -}} +{{- else }} + DB_CONNECTION: {{ $connection | b64enc -}} +{{- end }} +{{- end }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: grafana-db +type: Opaque +data: + my.cnf: {{ tuple "secrets/_my.cnf.tpl" . | include "helm-toolkit.utils.template" | b64enc }} +{{- end }} diff --git a/grafana/templates/secret-ingress-tls.yaml b/grafana/templates/secret-ingress-tls.yaml new file mode 100644 index 0000000000..f77ffb6045 --- /dev/null +++ b/grafana/templates/secret-ingress-tls.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_ingress_tls }} +{{- include "helm-toolkit.manifests.secret_ingress_tls" ( dict "envAll" . "backendServiceType" "grafana" "backendService" "grafana" ) }} +{{- end }} diff --git a/grafana/templates/secret-prom-creds.yaml b/grafana/templates/secret-prom-creds.yaml new file mode 100644 index 0000000000..a0a7d25c8d --- /dev/null +++ b/grafana/templates/secret-prom-creds.yaml @@ -0,0 +1,30 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_prom_creds }} +{{- $envAll := . }} +{{- $secretName := index $envAll.Values.secrets.prometheus.user }} + +{{- $prometheus_user := .Values.endpoints.monitoring.auth.user.username }} +{{- $prometheus_password := .Values.endpoints.monitoring.auth.user.password }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} +type: Opaque +data: + PROMETHEUS_USERNAME: {{ .Values.endpoints.monitoring.auth.user.username | b64enc }} + PROMETHEUS_PASSWORD: {{ .Values.endpoints.monitoring.auth.user.password | b64enc }} +{{- end }} diff --git a/grafana/templates/secret-registry.yaml b/grafana/templates/secret-registry.yaml new file mode 100644 index 0000000000..da979b3223 --- /dev/null +++ b/grafana/templates/secret-registry.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_registry .Values.endpoints.oci_image_registry.auth.enabled }} +{{ include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) }} +{{- end }} diff --git a/grafana/templates/secrets/_my.cnf.tpl b/grafana/templates/secrets/_my.cnf.tpl new file mode 100644 index 0000000000..ca7acfec7e --- /dev/null +++ b/grafana/templates/secrets/_my.cnf.tpl @@ -0,0 +1,17 @@ +{{/* + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */}} + + [client] + user = {{ .Values.endpoints.oslo_db.auth.admin.username }} + password = {{ .Values.endpoints.oslo_db.auth.admin.password }} diff --git a/grafana/templates/service-ingress.yaml b/grafana/templates/service-ingress.yaml new file mode 100644 index 0000000000..b4f3d29219 --- /dev/null +++ b/grafana/templates/service-ingress.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.service_ingress .Values.network.grafana.ingress.public }} +{{- $serviceIngressOpts := dict "envAll" . "backendServiceType" "grafana" -}} +{{ $serviceIngressOpts | include "helm-toolkit.manifests.service_ingress" }} +{{- end }} diff --git a/grafana/templates/service.yaml b/grafana/templates/service.yaml new file mode 100644 index 0000000000..6bd43a74a9 --- /dev/null +++ b/grafana/templates/service.yaml @@ -0,0 +1,34 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.service }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "grafana" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +spec: + ports: + - name: dashboard + port: {{ tuple "grafana" "internal" "grafana" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + {{ if .Values.network.grafana.node_port.enabled }} + nodePort: {{ .Values.network.grafana.node_port.port }} + {{ end }} + selector: +{{ tuple $envAll "grafana" "dashboard" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + {{ if .Values.network.grafana.node_port.enabled }} + type: NodePort + {{ end }} +{{- end }} diff --git a/grafana/values.yaml b/grafana/values.yaml new file mode 100644 index 0000000000..d13c944fec --- /dev/null +++ b/grafana/values.yaml @@ -0,0 +1,554 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for grafana +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +--- +images: + tags: + grafana: docker.io/grafana/grafana:9.2.10 + mariadb: docker.io/openstackhelm/mariadb:latest-ubuntu_focal + dep_check: quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal + db_init: docker.io/openstackhelm/heat:2024.1-ubuntu_jammy + grafana_db_session_sync: docker.io/openstackhelm/heat:2024.1-ubuntu_jammy + selenium_tests: docker.io/openstackhelm/osh-selenium:latest-ubuntu_jammy + image_repo_sync: docker.io/library/docker:17.07.0 + grafana_image_renderer: docker.io/grafana/grafana-image-renderer:3.10.5 + pull_policy: IfNotPresent + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +# Use selenium v4 syntax +selenium_v4: true + +labels: + grafana: + node_selector_key: openstack-control-plane + node_selector_value: enabled + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + test: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +pod: + env: + grafana: null + grafana_run_migrator: + GF_DEFAULT_FORCE_MIGRATION: false + security_context: + dashboard: + pod: + runAsUser: 472 + container: + grafana: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + db_init: + pod: + runAsUser: 472 + container: + grafana_db_init_session: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + grafana_db_init: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + db_session_sync: + pod: + runAsUser: 472 + container: + grafana_db_session_sync: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + set_admin_user: + pod: + runAsUser: 472 + container: + grafana_set_admin_password: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + run_migrator: + pod: + runAsUser: 472 + container: + prepare_grafana_migrator: + runAsUser: 0 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + grafana_run_migrator: + runAsUser: 65534 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + grafana_set_admin_password: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + test: + pod: + runAsUser: 0 + container: + helm_tests: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + weight: + default: 10 + mounts: + grafana: + init_container: null + grafana: + replicas: + grafana: 1 + lifecycle: + upgrades: + deployments: + revision_history: 3 + pod_replacement_strategy: RollingUpdate + rolling_update: + max_unavailable: 1 + max_surge: 3 + termination_grace_period: + grafana: + timeout: 600 + resources: + enabled: false + jobs: + image_repo_sync: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + bootstrap: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + db_init: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + db_init_session: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + grafana_db_session_sync: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + set_admin_user: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + run_migrator: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + tests: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + grafana: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + grafana: + username: grafana + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + oslo_db: + namespace: null + auth: + admin: + username: root + password: password + secret: + tls: + internal: mariadb-tls-direct + user: + username: grafana + password: password + hosts: + default: mariadb + host_fqdn_override: + default: null + path: /grafana + scheme: mysql+pymysql + port: + mysql: + default: 3306 + oslo_db_session: + namespace: null + auth: + admin: + username: root + password: password + secret: + tls: + internal: mariadb-tls-direct + user: + username: grafana_session + password: password + hosts: + default: mariadb + host_fqdn_override: + default: null + path: /grafana_session + scheme: mysql+pymysql + port: + mysql: + default: 3306 + grafana: + name: grafana + namespace: null + auth: + admin: + username: admin + password: password + hosts: + default: grafana-dashboard + public: grafana + host_fqdn_override: + default: null + path: + default: null + scheme: + default: http + port: + grafana: + default: 3000 + public: 80 + image_rendering: 8081 + monitoring: + name: prometheus + namespace: null + auth: + user: + username: admin + password: changeme + hosts: + default: prom-metrics + public: prometheus + host_fqdn_override: + default: null + path: + default: null + scheme: + default: http + port: + api: + default: 80 + public: 80 + ldap: + hosts: + default: ldap + auth: + admin: + bind_dn: "cn=admin,dc=cluster,dc=local" + password: password + host_fqdn_override: + default: null + path: + default: "ou=People,dc=cluster,dc=local" + scheme: + default: ldap + port: + ldap: + default: 389 + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - grafana-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + db_init: + services: + - endpoint: internal + service: oslo_db + db_init_session: + services: + - endpoint: internal + service: oslo_db + db_session_sync: + jobs: + - grafana-db-init-session + services: + - endpoint: internal + service: oslo_db + grafana: + jobs: + - grafana-db-init + - grafana-db-session-sync + - grafana-set-admin-user + - grafana-run-migrator + services: + - endpoint: internal + service: oslo_db + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + set_admin_user: + jobs: + - grafana-db-init + services: + - endpoint: internal + service: oslo_db + run_migrator: + jobs: + - grafana-set-admin-user + services: + - endpoint: internal + service: oslo_db + tests: + services: + - endpoint: internal + service: grafana + +network: + grafana: + node_port: + enabled: false + port: 30902 + ingress: + public: true + classes: + namespace: "nginx" + cluster: "nginx-cluster" + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / + +network_policy: + grafana: + ingress: + - {} + egress: + - {} + +secrets: + oci_image_registry: + grafana: grafana-oci-image-registry-key + oslo_db: + admin: grafana-db-admin + user: grafana-db-user + oslo_db_session: + admin: grafana-session-db-admin + user: grafana-session-db-user + tls: + grafana: + grafana: + public: grafana-tls-public + prometheus: + user: prometheus-user-creds + +manifests: + certificates: false + configmap_bin: true + configmap_etc: true + configmap_dashboards: true + deployment: true + ingress: true + helm_tests: true + job_db_init: true + job_db_init_session: true + job_db_session_sync: true + job_image_repo_sync: true + job_set_admin_user: true + job_run_migrator: true + network_policy: false + secret_db: true + secret_db_session: true + secret_admin_creds: true + secret_ingress_tls: true + secret_prom_creds: true + secret_registry: true + service: true + service_ingress: true + +conf: + ldap: + config: + base_dns: + search: "dc=cluster,dc=local" + group_search: "ou=Groups,dc=cluster,dc=local" + filters: + search: "(uid=%s)" + group_search: "(&(objectclass=posixGroup)(memberUID=uid=%s,ou=People,dc=cluster,dc=local))" + template: | + verbose_logging = false + [[servers]] + host = "{{ tuple "ldap" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }}" + port = {{ tuple "ldap" "internal" "ldap" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + use_ssl = false + start_tls = false + ssl_skip_verify = false + bind_dn = "{{ .Values.endpoints.ldap.auth.admin.bind_dn }}" + bind_password = '{{ .Values.endpoints.ldap.auth.admin.password }}' + search_filter = "{{ .Values.conf.ldap.config.filters.search }}" + search_base_dns = ["{{ .Values.conf.ldap.config.base_dns.search }}"] + group_search_filter = "{{ .Values.conf.ldap.config.filters.group_search }}" + group_search_base_dns = ["{{ .Values.conf.ldap.config.base_dns.group_search }}"] + [servers.attributes] + username = "uid" + surname = "sn" + member_of = "cn" + email = "mail" + [[servers.group_mappings]] + group_dn = "{{.Values.endpoints.ldap.auth.admin.bind_dn }}" + org_role = "Admin" + [[servers.group_mappings]] + group_dn = "*" + org_role = "Viewer" + provisioning: + dashboards: + apiVersion: 1 + providers: + - name: 'osh-infra-dashboards' + orgId: 1 + folder: '' + type: file + disableDeletion: false + editable: false + options: + path: /etc/grafana/dashboards + datasources: + template: | + apiVersion: 1 + datasources: + - name: prometheus + type: prometheus + access: proxy + orgId: 1 + editable: true + basicAuth: true + basicAuthUser: {{ .Values.endpoints.monitoring.auth.user.username }} + secureJsonData: + basicAuthPassword: {{ .Values.endpoints.monitoring.auth.user.password }} + url: {{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" }} + grafana: + alerting: + enabled: false + unified_alerting: + enabled: true + image_rendering_sidecar: + enabled: false + # https://kubernetes.io/docs/concepts/workloads/pods/sidecar-containers/ + k8s_sidecar_feature_enabled: true + analytics: + reporting_enabled: false + check_for_updates: false + auth.ldap: + enabled: true + config_file: /etc/grafana/ldap.toml + paths: + data: /var/lib/grafana/data + plugins: /var/lib/grafana/plugins + alerting: /var/lib/grafana/alerting + csv: /var/lib/grafana/csv + provisioning: /etc/grafana/provisioning + server: + protocol: http + http_port: 3000 + database: + type: mysql + session: + provider: mysql + provider_config: null + cookie_name: grafana_sess + cookie_secure: false + session_life_time: 86400 + security: + admin_user: ${GF_SECURITY_ADMIN_USER} + admin_password: ${GF_SECURITY_ADMIN_PASSWORD} + cookie_username: grafana_user + cookie_remember_name: grafana_remember + login_remember_days: 7 + dashboards: + default_home_dashboard_path: /etc/grafana/dashboards/home_dashboard.json + users: + allow_sign_up: false + allow_org_create: false + auto_assign_org: true + default_theme: dark + log: + mode: console + level: info + grafana_net: + url: https://grafana.net + dashboards: {} +... diff --git a/helm-toolkit/Chart.yaml b/helm-toolkit/Chart.yaml new file mode 100644 index 0000000000..b84ba12b48 --- /dev/null +++ b/helm-toolkit/Chart.yaml @@ -0,0 +1,27 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: v1.0.0 +description: OpenStack-Helm Helm-Toolkit +name: helm-toolkit +version: 2024.2.0 +home: https://docs.openstack.org/openstack-helm +icon: https://www.openstack.org/themes/openstack/images/project-mascots/OpenStack-Helm/OpenStack_Project_OpenStackHelm_vertical.png +sources: + - https://opendev.org/openstack/openstack-helm-infra + - https://opendev.org/openstack/openstack-helm +maintainers: + - name: OpenStack-Helm Authors +dependencies: [] +... diff --git a/helm-toolkit/templates/endpoints/_authenticated_endpoint_uri_lookup.tpl b/helm-toolkit/templates/endpoints/_authenticated_endpoint_uri_lookup.tpl new file mode 100644 index 0000000000..d7390d8bed --- /dev/null +++ b/helm-toolkit/templates/endpoints/_authenticated_endpoint_uri_lookup.tpl @@ -0,0 +1,58 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Resolves database, or basic auth, style endpoints +values: | + endpoints: + cluster_domain_suffix: cluster.local + oslo_db: + auth: + admin: + username: root + password: password + service_username: + username: username + password: password + hosts: + default: mariadb + host_fqdn_override: + default: null + path: /dbname + scheme: mysql+pymysql + port: + mysql: + default: 3306 +usage: | + {{ tuple "oslo_db" "internal" "service_username" "mysql" . | include "helm-toolkit.endpoints.authenticated_endpoint_uri_lookup" }} +return: | + mysql+pymysql://serviceuser:password@mariadb.default.svc.cluster.local:3306/dbname +*/}} + +{{- define "helm-toolkit.endpoints.authenticated_endpoint_uri_lookup" -}} +{{- $type := index . 0 -}} +{{- $endpoint := index . 1 -}} +{{- $userclass := index . 2 -}} +{{- $port := index . 3 -}} +{{- $context := index . 4 -}} +{{- $endpointScheme := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" }} +{{- $userMap := index $context.Values.endpoints ( $type | replace "-" "_" ) "auth" $userclass }} +{{- $endpointUser := index $userMap "username" }} +{{- $endpointPass := index $userMap "password" | urlquery }} +{{- $endpointHost := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} +{{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +{{- $endpointPath := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.keystone_endpoint_path_lookup" }} +{{- printf "%s://%s:%s@%s:%s%s" $endpointScheme $endpointUser $endpointPass $endpointHost $endpointPort $endpointPath -}} +{{- end -}} diff --git a/helm-toolkit/templates/endpoints/_authenticated_transport_endpoint_uri_lookup.tpl b/helm-toolkit/templates/endpoints/_authenticated_transport_endpoint_uri_lookup.tpl new file mode 100644 index 0000000000..b9ac9d9ab4 --- /dev/null +++ b/helm-toolkit/templates/endpoints/_authenticated_transport_endpoint_uri_lookup.tpl @@ -0,0 +1,121 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Resolves endpoint string suitible for use with oslo.messaging transport url + See: https://docs.openstack.org/oslo.messaging/latest/reference/transport.html#oslo_messaging.TransportURL +examples: + - values: | + endpoints: + cluster_domain_suffix: cluster.local + oslo_messaging: + auth: + cinder: + username: cinder + password: password + statefulset: + replicas: 2 + name: rabbitmq-rabbitmq + hosts: + default: rabbitmq + host_fqdn_override: + default: null + path: /cinder + scheme: rabbit + port: + amqp: + default: 5672 + usage: | + {{ tuple "oslo_messaging" "internal" "cinder" "amqp" . | include "helm-toolkit.endpoints.authenticated_transport_endpoint_uri_lookup" }} + return: | + rabbit://cinder:password@rabbitmq-rabbitmq-0.rabbitmq.default.svc.cluster.local:5672,cinder:password@rabbitmq-rabbitmq-1.rabbitmq.default.svc.cluster.local:5672/cinder + - values: | + endpoints: + cluster_domain_suffix: cluster.local + oslo_messaging: + auth: + cinder: + username: cinder + password: password + statefulset: null + hosts: + default: rabbitmq + host_fqdn_override: + default: null + path: /cinder + scheme: rabbit + port: + amqp: + default: 5672 + usage: | + {{ tuple "oslo_messaging" "internal" "cinder" "amqp" . | include "helm-toolkit.endpoints.authenticated_transport_endpoint_uri_lookup" }} + return: | + rabbit://cinder:password@rabbitmq.default.svc.cluster.local:5672/cinder + - values: | + endpoints: + cluster_domain_suffix: cluster.local + oslo_messaging: + auth: + cinder: + username: cinder + password: password + statefulset: + replicas: 2 + name: rabbitmq-rabbitmq + hosts: + default: rabbitmq + host_fqdn_override: + default: rabbitmq.openstackhelm.org + path: /cinder + scheme: rabbit + port: + amqp: + default: 5672 + usage: | + {{ tuple "oslo_messaging" "internal" "cinder" "amqp" . | include "helm-toolkit.endpoints.authenticated_transport_endpoint_uri_lookup" }} + return: | + rabbit://cinder:password@rabbitmq.openstackhelm.org:5672/cinder +*/}} + +{{- define "helm-toolkit.endpoints.authenticated_transport_endpoint_uri_lookup" -}} +{{- $type := index . 0 -}} +{{- $endpoint := index . 1 -}} +{{- $userclass := index . 2 -}} +{{- $port := index . 3 -}} +{{- $context := index . 4 -}} +{{- $endpointScheme := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" }} +{{- $userMap := index $context.Values.endpoints ( $type | replace "-" "_" ) "auth" $userclass }} +{{- $ssMap := index $context.Values.endpoints ( $type | replace "-" "_" ) "statefulset" | default false}} +{{- $hostFqdnOverride := index $context.Values.endpoints ( $type | replace "-" "_" ) "host_fqdn_override" }} +{{- $endpointUser := index $userMap "username" }} +{{- $endpointPass := index $userMap "password" | urlquery }} +{{- $endpointHostSuffix := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} +{{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +{{- $local := dict "endpointCredsAndHosts" list -}} +{{- if not (or (index $hostFqdnOverride $endpoint | default ( index $hostFqdnOverride "default" ) ) ( not $ssMap ) ) }} +{{- $endpointHostPrefix := $ssMap.name }} +{{- range $podInt := until ( atoi (print $ssMap.replicas ) ) }} +{{- $endpointCredAndHost := printf "%s:%s@%s-%d.%s:%s" $endpointUser $endpointPass $endpointHostPrefix $podInt $endpointHostSuffix $endpointPort }} +{{- $_ := set $local "endpointCredsAndHosts" ( append $local.endpointCredsAndHosts $endpointCredAndHost ) }} +{{- end }} +{{- else }} +{{- $endpointHost := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} +{{- $endpointCredAndHost := printf "%s:%s@%s:%s" $endpointUser $endpointPass $endpointHost $endpointPort }} +{{- $_ := set $local "endpointCredsAndHosts" ( append $local.endpointCredsAndHosts $endpointCredAndHost ) }} +{{- end }} +{{- $endpointCredsAndHosts := include "helm-toolkit.utils.joinListWithComma" $local.endpointCredsAndHosts }} +{{- $endpointPath := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.keystone_endpoint_path_lookup" }} +{{- printf "%s://%s%s" $endpointScheme $endpointCredsAndHosts $endpointPath }} +{{- end -}} diff --git a/helm-toolkit/templates/endpoints/_endpoint_host_lookup.tpl b/helm-toolkit/templates/endpoints/_endpoint_host_lookup.tpl new file mode 100644 index 0000000000..fb8bbe7d39 --- /dev/null +++ b/helm-toolkit/templates/endpoints/_endpoint_host_lookup.tpl @@ -0,0 +1,90 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Resolves either the fully qualified hostname, of if defined in the host field + IPv4 for an endpoint. +examples: + - values: | + endpoints: + cluster_domain_suffix: cluster.local + oslo_db: + hosts: + default: mariadb + host_fqdn_override: + default: null + usage: | + {{ tuple "oslo_db" "internal" . | include "helm-toolkit.endpoints.endpoint_host_lookup" }} + return: | + mariadb.default.svc.cluster.local + - values: | + endpoints: + cluster_domain_suffix: cluster.local + oslo_db: + hosts: + default: + host: mariadb + host_fqdn_override: + default: null + usage: | + {{ tuple "oslo_db" "internal" . | include "helm-toolkit.endpoints.endpoint_host_lookup" }} + return: | + mariadb.default.svc.cluster.local + - values: | + endpoints: + cluster_domain_suffix: cluster.local + oslo_db: + hosts: + default: 127.0.0.1 + host_fqdn_override: + default: null + usage: | + {{ tuple "oslo_db" "internal" . | include "helm-toolkit.endpoints.endpoint_host_lookup" }} + return: | + 127.0.0.1 + - values: | + endpoints: + cluster_domain_suffix: cluster.local + oslo_db: + hosts: + default: + host: 127.0.0.1 + host_fqdn_override: + default: null + usage: | + {{ tuple "oslo_db" "internal" . | include "helm-toolkit.endpoints.endpoint_host_lookup" }} + return: | + 127.0.0.1 +*/}} + +{{- define "helm-toolkit.endpoints.endpoint_host_lookup" -}} +{{- $type := index . 0 -}} +{{- $endpoint := index . 1 -}} +{{- $context := index . 2 -}} +{{- $endpointMap := index $context.Values.endpoints ( $type | replace "-" "_" ) }} +{{- $endpointScheme := $endpointMap.scheme }} +{{- $_ := set $context.Values "__endpointHost" ( index $endpointMap.hosts $endpoint | default $endpointMap.hosts.default ) }} +{{- if kindIs "map" $context.Values.__endpointHost }} +{{- $_ := set $context.Values "__endpointHost" ( index $context.Values.__endpointHost "host" ) }} +{{- end }} +{{- $endpointHost := $context.Values.__endpointHost }} +{{- if regexMatch "[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+" $endpointHost }} +{{- $endpointHostname := printf "%s" $endpointHost }} +{{- printf "%s" $endpointHostname -}} +{{- else }} +{{- $endpointHostname := tuple $type $endpoint $context | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }} +{{- printf "%s" $endpointHostname -}} +{{- end }} +{{- end -}} diff --git a/helm-toolkit/templates/endpoints/_endpoint_port_lookup.tpl b/helm-toolkit/templates/endpoints/_endpoint_port_lookup.tpl new file mode 100644 index 0000000000..447efe7661 --- /dev/null +++ b/helm-toolkit/templates/endpoints/_endpoint_port_lookup.tpl @@ -0,0 +1,41 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Resolves the port for an endpoint +values: | + endpoints: + cluster_domain_suffix: cluster.local + oslo_db: + port: + mysql: + default: 3306 +usage: | + {{ tuple "oslo_db" "internal" "mysql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +return: | + 3306 +*/}} + +{{- define "helm-toolkit.endpoints.endpoint_port_lookup" -}} +{{- $type := index . 0 -}} +{{- $endpoint := index . 1 -}} +{{- $port := index . 2 -}} +{{- $context := index . 3 -}} +{{- $typeYamlSafe := $type | replace "-" "_" }} +{{- $endpointMap := index $context.Values.endpoints $typeYamlSafe }} +{{- $endpointPortMAP := index $endpointMap.port $port }} +{{- $endpointPort := index $endpointPortMAP $endpoint | default ( index $endpointPortMAP "default" ) }} +{{- printf "%1.f" $endpointPort -}} +{{- end -}} diff --git a/helm-toolkit/templates/endpoints/_endpoint_token_lookup.tpl b/helm-toolkit/templates/endpoints/_endpoint_token_lookup.tpl new file mode 100644 index 0000000000..3a268c0f77 --- /dev/null +++ b/helm-toolkit/templates/endpoints/_endpoint_token_lookup.tpl @@ -0,0 +1,36 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Gets the token for an endpoint +values: | + endpoints: + keystone: + auth: + admin: + token: zh78JzXgw6YUKy2e +usage: | + {{ tuple "keystone" "admin" . | include "helm-toolkit.endpoints.endpoint_token_lookup" }} +return: | + zh78JzXgw6YUKy2e +*/}} + +{{- define "helm-toolkit.endpoints.endpoint_token_lookup" -}} +{{- $type := index . 0 -}} +{{- $userName := index . 1 -}} +{{- $context := index . 2 -}} +{{- $serviceToken := index $context.Values.endpoints ( $type | replace "-" "_" ) "auth" $userName "token" }} +{{- printf "%s" $serviceToken -}} +{{- end -}} diff --git a/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl b/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl new file mode 100644 index 0000000000..728b994358 --- /dev/null +++ b/helm-toolkit/templates/endpoints/_host_and_port_endpoint_uri_lookup.tpl @@ -0,0 +1,89 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Resolves 'hostname:port' for an endpoint, or several hostname:port pairs for statefulset e.g + 'hostname1:port1,hostname2:port2,hostname3:port3', +examples: + - values: | + endpoints: + cluster_domain_suffix: cluster.local + oslo_db: + hosts: + default: mariadb + host_fqdn_override: + default: null + port: + mysql: + default: 3306 + usage: | + {{ tuple "oslo_db" "internal" "mysql" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} + return: | + mariadb.default.svc.cluster.local:3306 + - values: | + endpoints: + cluster_domain_suffix: cluster.local + oslo_db: + hosts: + default: 127.0.0.1 + host_fqdn_override: + default: null + port: + mysql: + default: 3306 + usage: | + {{ tuple "oslo_db" "internal" "mysql" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} + return: | + 127.0.0.1:3306 + - values: | + endpoints: + oslo_cache: + hosts: + default: memcached + host_fqdn_override: + default: null + statefulset: + name: openstack-memcached-memcached + replicas: 3 + port: + memcache: + default: 11211 + usage: | + {{ tuple "oslo_cache" "internal" "memcache" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} + return: | + openstack-memcached-memcached-0:11211,openstack-memcached-memcached-1:11211,openstack-memcached-memcached-2:11211 +*/}} + +{{- define "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" -}} +{{- $type := index . 0 -}} +{{- $endpoint := index . 1 -}} +{{- $port := index . 2 -}} +{{- $context := index . 3 -}} +{{- $ssMap := index $context.Values.endpoints ( $type | replace "-" "_" ) "statefulset" | default false -}} +{{- $local := dict "endpointHosts" list -}} +{{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" -}} +{{- if $ssMap -}} +{{- $endpointHostPrefix := $ssMap.name -}} +{{- $endpointHostSuffix := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} +{{- range $podInt := until ( atoi (print $ssMap.replicas ) ) -}} +{{- $endpointHostname := printf "%s-%d.%s:%s" $endpointHostPrefix $podInt $endpointHostSuffix $endpointPort -}} +{{- $_ := set $local "endpointHosts" ( append $local.endpointHosts $endpointHostname ) -}} +{{- end -}} +{{- else -}} +{{- $endpointHostname := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" -}} +{{- $_ := set $local "endpointHosts" ( append $local.endpointHosts (printf "%s:%s" $endpointHostname $endpointPort) ) -}} +{{- end -}} +{{ include "helm-toolkit.utils.joinListWithComma" $local.endpointHosts }} +{{- end -}} diff --git a/helm-toolkit/templates/endpoints/_hostname_fqdn_endpoint_lookup.tpl b/helm-toolkit/templates/endpoints/_hostname_fqdn_endpoint_lookup.tpl new file mode 100644 index 0000000000..26374e348a --- /dev/null +++ b/helm-toolkit/templates/endpoints/_hostname_fqdn_endpoint_lookup.tpl @@ -0,0 +1,76 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Resolves the fully qualified hostname for an endpoint +examples: + - values: | + endpoints: + cluster_domain_suffix: cluster.local + oslo_db: + hosts: + default: mariadb + host_fqdn_override: + default: null + usage: | + {{ tuple "oslo_db" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }} + return: | + mariadb.default.svc.cluster.local + - values: | + endpoints: + cluster_domain_suffix: cluster.local + oslo_db: + hosts: + default: mariadb + host_fqdn_override: + default: mariadb.openstackhelm.openstack.org + usage: | + {{ tuple "oslo_db" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }} + return: | + mariadb.openstackhelm.openstack.org + - values: | + endpoints: + cluster_domain_suffix: cluster.local + oslo_db: + hosts: + default: mariadb + host_fqdn_override: + default: + host: mariadb.openstackhelm.openstack.org + usage: | + {{ tuple "oslo_db" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }} + return: | + mariadb.openstackhelm.openstack.org +*/}} + +{{- define "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" -}} +{{- $type := index . 0 -}} +{{- $endpoint := index . 1 -}} +{{- $context := index . 2 -}} +{{- $endpointMap := index $context.Values.endpoints ( $type | replace "-" "_" ) }} +{{- $endpointHostNamespaced := tuple $type $endpoint $context | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" }} +{{- $endpointClusterHostname := printf "%s.svc.%s" $endpointHostNamespaced $context.Values.endpoints.cluster_domain_suffix }} +{{- $_ := set $context.Values "__FQDNendpointHostDefault" ( index $endpointMap.host_fqdn_override "default" | default "" ) }} +{{- if kindIs "map" $context.Values.__FQDNendpointHostDefault }} +{{- $_ := set $context.Values "__FQDNendpointHostDefault" ( index $context.Values.__FQDNendpointHostDefault "host" ) }} +{{- end }} +{{- if kindIs "map" (index $endpointMap.host_fqdn_override $endpoint) }} +{{- $endpointHostname := index $endpointMap.host_fqdn_override $endpoint "host" | default $context.Values.__FQDNendpointHostDefault | default $endpointClusterHostname }} +{{- printf "%s" $endpointHostname -}} +{{- else }} +{{- $endpointHostname := index $endpointMap.host_fqdn_override $endpoint | default $context.Values.__FQDNendpointHostDefault | default $endpointClusterHostname }} +{{- printf "%s" $endpointHostname -}} +{{- end -}} +{{- end -}} diff --git a/helm-toolkit/templates/endpoints/_hostname_namespaced_endpoint_lookup.tpl b/helm-toolkit/templates/endpoints/_hostname_namespaced_endpoint_lookup.tpl new file mode 100644 index 0000000000..9d60393770 --- /dev/null +++ b/helm-toolkit/templates/endpoints/_hostname_namespaced_endpoint_lookup.tpl @@ -0,0 +1,40 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Resolves the namespace scoped hostname for an endpoint +values: | + endpoints: + oslo_db: + hosts: + default: mariadb + host_fqdn_override: + default: null +usage: | + {{ tuple "oslo_db" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" }} +return: | + mariadb.default +*/}} + +{{- define "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" -}} +{{- $type := index . 0 -}} +{{- $endpoint := index . 1 -}} +{{- $context := index . 2 -}} +{{- $endpointMap := index $context.Values.endpoints ( $type | replace "-" "_" ) }} +{{- $namespace := $endpointMap.namespace | default $context.Release.Namespace }} +{{- $endpointHost := tuple $type $endpoint $context | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +{{- $endpointClusterHostname := printf "%s.%s" $endpointHost $namespace }} +{{- printf "%s" $endpointClusterHostname -}} +{{- end -}} diff --git a/helm-toolkit/templates/endpoints/_hostname_namespaced_endpoint_namespace_lookup.tpl b/helm-toolkit/templates/endpoints/_hostname_namespaced_endpoint_namespace_lookup.tpl new file mode 100644 index 0000000000..cc4d4de622 --- /dev/null +++ b/helm-toolkit/templates/endpoints/_hostname_namespaced_endpoint_namespace_lookup.tpl @@ -0,0 +1,38 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Resolves the namespace scoped hostname for an endpoint +values: | + endpoints: + oslo_db: + hosts: + default: mariadb + host_fqdn_override: + default: null +usage: | + {{ tuple "oslo_db" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_namespace_lookup" }} +return: | + default +*/}} + +{{- define "helm-toolkit.endpoints.hostname_namespaced_endpoint_namespace_lookup" -}} +{{- $type := index . 0 -}} +{{- $endpoint := index . 1 -}} +{{- $context := index . 2 -}} +{{- $endpointMap := index $context.Values.endpoints ( $type | replace "-" "_" ) }} +{{- $namespace := $endpointMap.namespace | default $context.Release.Namespace }} +{{- printf "%s" $namespace -}} +{{- end -}} diff --git a/helm-toolkit/templates/endpoints/_hostname_short_endpoint_lookup.tpl b/helm-toolkit/templates/endpoints/_hostname_short_endpoint_lookup.tpl new file mode 100644 index 0000000000..f23c624f53 --- /dev/null +++ b/helm-toolkit/templates/endpoints/_hostname_short_endpoint_lookup.tpl @@ -0,0 +1,61 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Resolves the short hostname for an endpoint +examples: + - values: | + endpoints: + oslo_db: + hosts: + default: mariadb + host_fqdn_override: + default: null + usage: | + {{ tuple "oslo_db" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + return: | + mariadb + - values: | + endpoints: + oslo_db: + hosts: + default: + host: mariadb + host_fqdn_override: + default: null + usage: | + {{ tuple "oslo_db" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + return: | + mariadb +*/}} + +{{- define "helm-toolkit.endpoints.hostname_short_endpoint_lookup" -}} +{{- $type := index . 0 -}} +{{- $endpoint := index . 1 -}} +{{- $context := index . 2 -}} +{{- $endpointMap := index $context.Values.endpoints ( $type | replace "-" "_" ) }} +{{- $endpointScheme := $endpointMap.scheme }} +{{- $_ := set $context.Values "__endpointHost" ( index $endpointMap.hosts $endpoint | default $endpointMap.hosts.default ) }} +{{- if kindIs "map" $context.Values.__endpointHost }} +{{- $_ := set $context.Values "__endpointHost" ( index $context.Values.__endpointHost "host" ) }} +{{- end }} +{{- $endpointHost := $context.Values.__endpointHost }} +{{- if regexMatch "[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+" $endpointHost }} +{{- printf "%s" $type -}} +{{- else }} +{{- $endpointHostname := printf "%s" $endpointHost }} +{{- printf "%s" $endpointHostname -}} +{{- end }} +{{- end -}} diff --git a/helm-toolkit/templates/endpoints/_keystone_endpoint_name_lookup.tpl b/helm-toolkit/templates/endpoints/_keystone_endpoint_name_lookup.tpl new file mode 100644 index 0000000000..e31c0ebe6e --- /dev/null +++ b/helm-toolkit/templates/endpoints/_keystone_endpoint_name_lookup.tpl @@ -0,0 +1,34 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Resolves the service name for an service type +values: | + endpoints: + identity: + name: keystone +usage: | + {{ tuple identity . | include "keystone_endpoint_name_lookup" }} +return: | + "keystone" +*/}} + +{{- define "helm-toolkit.endpoints.keystone_endpoint_name_lookup" -}} +{{- $type := index . 0 -}} +{{- $context := index . 1 -}} +{{- $endpointMap := index $context.Values.endpoints ( $type | replace "-" "_" ) }} +{{- $endpointName := index $endpointMap "name" }} +{{- $endpointName | quote -}} +{{- end -}} diff --git a/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl b/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl new file mode 100644 index 0000000000..24eb569427 --- /dev/null +++ b/helm-toolkit/templates/endpoints/_keystone_endpoint_path_lookup.tpl @@ -0,0 +1,48 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# FIXME(portdirect): it appears the port input here serves no purpose, +# and should be removed. In addition this function is bugged, do we use it? + +{{/* +abstract: | + Resolves the path for an endpoint +values: | + endpoints: + cluster_domain_suffix: cluster.local + oslo_db: + path: + default: /dbname + port: + mysql: + default: 3306 +usage: | + {{ tuple "oslo_db" "internal" "mysql" . | include "helm-toolkit.endpoints.keystone_endpoint_path_lookup" }} +return: | + /dbname +*/}} + +{{- define "helm-toolkit.endpoints.keystone_endpoint_path_lookup" -}} +{{- $type := index . 0 -}} +{{- $endpoint := index . 1 -}} +{{- $port := index . 2 -}} +{{- $context := index . 3 -}} +{{- $endpointMap := index $context.Values.endpoints ( $type | replace "-" "_" ) }} +{{- if kindIs "string" $endpointMap.path }} +{{- printf "%s" $endpointMap.path | default "" -}} +{{- else -}} +{{- $endpointPath := index $endpointMap.path $endpoint | default $endpointMap.path.default | default "" }} +{{- printf "%s" $endpointPath -}} +{{- end -}} +{{- end -}} diff --git a/helm-toolkit/templates/endpoints/_keystone_endpoint_scheme_lookup.tpl b/helm-toolkit/templates/endpoints/_keystone_endpoint_scheme_lookup.tpl new file mode 100644 index 0000000000..b35cb0b747 --- /dev/null +++ b/helm-toolkit/templates/endpoints/_keystone_endpoint_scheme_lookup.tpl @@ -0,0 +1,55 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# FIXME(portdirect): it appears the port input here serves no purpose, +# and should be removed. In addition this function is bugged, do we use it? + +{{/* +abstract: | + Resolves the scheme for an endpoint +values: | + endpoints: + cluster_domain_suffix: cluster.local + oslo_db: + scheme: + default: + mysql+pymysql + port: + mysql: + default: 3306 +usage: | + {{ tuple "oslo_db" "internal" "mysql" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" }} +return: | + mysql+pymysql +*/}} + +# This function returns the scheme for a service, it takes an tuple +# input in the form: service-type, endpoint-class, port-name. eg: +# { tuple "etcd" "internal" "client" . | include "helm-toolkit.endpoints.keystone_scheme_lookup" } +# will return the scheme setting for this particular endpoint. In other words, for most endpoints +# it will return either 'http' or 'https' + +{{- define "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" -}} +{{- $type := index . 0 -}} +{{- $endpoint := index . 1 -}} +{{- $port := index . 2 -}} +{{- $context := index . 3 -}} +{{- $endpointMap := index $context.Values.endpoints ( $type | replace "-" "_" ) }} +{{- if kindIs "string" $endpointMap.scheme }} +{{- printf "%s" $endpointMap.scheme | default "http" -}} +{{- else -}} +{{- $endpointScheme := index $endpointMap.scheme $endpoint | default $endpointMap.scheme.default | default "http" }} +{{- printf "%s" $endpointScheme -}} +{{- end -}} +{{- end -}} diff --git a/helm-toolkit/templates/endpoints/_keystone_endpoint_uri_lookup.tpl b/helm-toolkit/templates/endpoints/_keystone_endpoint_uri_lookup.tpl new file mode 100644 index 0000000000..8d0819cd16 --- /dev/null +++ b/helm-toolkit/templates/endpoints/_keystone_endpoint_uri_lookup.tpl @@ -0,0 +1,52 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + This function helps resolve uri style endpoints. It will omit the port for + http when 80 is used, and 443 in the case of https. +values: | + endpoints: + cluster_domain_suffix: cluster.local + oslo_db: + hosts: + default: mariadb + host_fqdn_override: + default: null + path: /dbname + scheme: mysql+pymysql + port: + mysql: + default: 3306 +usage: | + {{ tuple "oslo_db" "internal" "mysql" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" }} +return: | + mysql+pymysql://mariadb.default.svc.cluster.local:3306/dbname +*/}} + +{{- define "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" -}} +{{- $type := index . 0 -}} +{{- $endpoint := index . 1 -}} +{{- $port := index . 2 -}} +{{- $context := index . 3 -}} +{{- $endpointScheme := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" }} +{{- $endpointHost := tuple $type $endpoint $context | include "helm-toolkit.endpoints.endpoint_host_lookup" }} +{{- $endpointPort := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +{{- $endpointPath := tuple $type $endpoint $port $context | include "helm-toolkit.endpoints.keystone_endpoint_path_lookup" }} +{{- if or ( and ( eq $endpointScheme "http" ) ( eq $endpointPort "80" ) ) ( and ( eq $endpointScheme "https" ) ( eq $endpointPort "443" ) ) -}} +{{- printf "%s://%s%s" $endpointScheme $endpointHost $endpointPath -}} +{{- else -}} +{{- printf "%s://%s:%s%s" $endpointScheme $endpointHost $endpointPort $endpointPath -}} +{{- end -}} +{{- end -}} diff --git a/helm-toolkit/templates/endpoints/_service_name_endpoint_with_namespace_lookup.tpl b/helm-toolkit/templates/endpoints/_service_name_endpoint_with_namespace_lookup.tpl new file mode 100644 index 0000000000..cf2ef3874d --- /dev/null +++ b/helm-toolkit/templates/endpoints/_service_name_endpoint_with_namespace_lookup.tpl @@ -0,0 +1,61 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + This function returns endpoint ":" pair from an endpoint + definition. This is used in kubernetes-entrypoint to support dependencies + between different services in different namespaces. + returns: the endpoint namespace and the service name, delimited by a colon + + Normally, the service name is constructed dynamically from the hostname + however when an ip address is used as the hostname, we default to + namespace:endpointCategoryName in order to construct a valid service name + however this can be overridden to a custom service name by defining + .service.name within the endpoint definition +values: | + endpoints: + cluster_domain_suffix: cluster.local + oslo_db: + namespace: foo + hosts: + default: mariadb + host_fqdn_override: + default: null +usage: | + {{ tuple oslo_db internal . | include "helm-toolkit.endpoints.service_name_endpoint_with_namespace_lookup" }} +return: | + foo:mariadb +*/}} + +{{- define "helm-toolkit.endpoints.service_name_endpoint_with_namespace_lookup" -}} +{{- $type := index . 0 -}} +{{- $endpoint := index . 1 -}} +{{- $context := index . 2 -}} +{{- $typeYamlSafe := $type | replace "-" "_" }} +{{- $endpointMap := index $context.Values.endpoints $typeYamlSafe }} +{{- with $endpointMap -}} +{{- $endpointName := index .hosts $endpoint | default .hosts.default }} +{{- $endpointNamespace := .namespace | default $context.Release.Namespace }} +{{- if regexMatch "[0-9]+\\.[0-9]+\\.[0-9]+\\.[0-9]+" $endpointName }} +{{- if .service.name }} +{{- printf "%s:%s" $endpointNamespace .service.name -}} +{{- else -}} +{{- printf "%s:%s" $endpointNamespace $typeYamlSafe -}} +{{- end -}} +{{- else -}} +{{- printf "%s:%s" $endpointNamespace $endpointName -}} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/helm-toolkit/templates/manifests/_ceph-storageclass.tpl b/helm-toolkit/templates/manifests/_ceph-storageclass.tpl new file mode 100644 index 0000000000..18453eef45 --- /dev/null +++ b/helm-toolkit/templates/manifests/_ceph-storageclass.tpl @@ -0,0 +1,111 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Creates a manifest for kubernete ceph storageclass +examples: + - values: | + manifests: + storageclass: true + storageclass: + rbd: + provision_storage_class: true + provisioner: "ceph.com/rbd" + metadata: + default_storage_class: true + name: general + parameters: + #We will grab the monitors value based on helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup + pool: rbd + admin_id: admin + ceph_configmap_name: "ceph-etc" + admin_secret_name: "pvc-ceph-conf-combined-storageclass" + admin_secret_namespace: ceph + user_id: admin + user_secret_name: "pvc-ceph-client-key" + image_format: "2" + image_features: layering + cephfs: + provision_storage_class: true + provisioner: "ceph.com/cephfs" + metadata: + name: cephfs + parameters: + admin_id: admin + admin_secret_name: "pvc-ceph-cephfs-client-key" + admin_secret_namespace: ceph + usage: | + {{- range $storageclass, $val := .Values.storageclass }} + {{ dict "storageclass_data" $val "envAll" $ | include "helm-toolkit.manifests.ceph-storageclass" }} + {{- end }} + return: | + --- + apiVersion: storage.k8s.io/v1 + kind: StorageClass + metadata: + annotations: + storageclass.kubernetes.io/is-default-class: "true" + name: general + provisioner: ceph.com/rbd + parameters: + monitors: ceph-mon..svc.:6789 + adminId: admin + adminSecretName: pvc-ceph-conf-combined-storageclass + adminSecretNamespace: ceph + pool: rbd + userId: admin + userSecretName: pvc-ceph-client-key + image_format: "2" + image_features: layering + --- + apiVersion: storage.k8s.io/v1 + kind: StorageClass + metadata: + name: cephfs + provisioner: ceph.com/cephfs + parameters: + monitors: ceph-mon..svc.:6789 + adminId: admin + adminSecretName: pvc-ceph-cephfs-client-key + adminSecretNamespace: ceph +*/}} + +{{- define "helm-toolkit.manifests.ceph-storageclass" -}} +{{- $envAll := index . "envAll" -}} +{{- $monHost := $envAll.Values.conf.ceph.global.mon_host -}} +{{- if empty $monHost -}} +{{- $monHost = tuple "ceph_mon" "internal" "mon" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" -}} +{{- end -}} +{{- $storageclassData := index . "storageclass_data" -}} +--- +{{- if $storageclassData.provision_storage_class }} +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: +{{- if $storageclassData.metadata.default_storage_class }} + annotations: + storageclass.kubernetes.io/is-default-class: "true" +{{- end }} + name: {{ $storageclassData.metadata.name }} +provisioner: {{ $storageclassData.provisioner }} +parameters: + monitors: {{ $monHost }} +{{- range $attr, $value := $storageclassData.parameters }} + {{ $attr }}: {{ $value | quote }} +{{- end }} +allowVolumeExpansion: true + +{{- end }} +{{- end }} diff --git a/helm-toolkit/templates/manifests/_certificates.tpl b/helm-toolkit/templates/manifests/_certificates.tpl new file mode 100644 index 0000000000..8be771e6ce --- /dev/null +++ b/helm-toolkit/templates/manifests/_certificates.tpl @@ -0,0 +1,108 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Creates a certificate using jetstack +examples: + - values: | + endpoints: + dashboard: + host_fqdn_override: + default: + host: null + tls: + secretName: keystone-tls-api + issuerRef: + name: ca-issuer + duration: 2160h + organization: + - ACME + commonName: keystone-api.openstack.svc.cluster.local + privateKey: + size: 2048 + usages: + - server auth + - client auth + dnsNames: + - cluster.local + issuerRef: + name: ca-issuer + usage: | + {{- $opts := dict "envAll" . "service" "dashboard" "type" "internal" -}} + {{ $opts | include "helm-toolkit.manifests.certificates" }} + return: | + --- + apiVersion: cert-manager.io/v1 + kind: Certificate + metadata: + name: keystone-tls-api + namespace: NAMESPACE + spec: + commonName: keystone-api.openstack.svc.cluster.local + dnsNames: + - cluster.local + duration: 2160h + issuerRef: + name: ca-issuer + privateKey: + size: 2048 + organization: + - ACME + secretName: keystone-tls-api + usages: + - server auth + - client auth +*/}} + +{{- define "helm-toolkit.manifests.certificates" -}} +{{- $envAll := index . "envAll" -}} +{{- $service := index . "service" -}} +{{- $type := index . "type" | default "" -}} +{{- $slice := index $envAll.Values.endpoints $service "host_fqdn_override" "default" "tls" -}} +{{/* Put in some sensible default value if one is not provided by values.yaml */}} +{{/* If a dnsNames list is not in the values.yaml, it can be overridden by a passed-in parameter. + This allows user to use other HTK method to determine the URI and pass that into this method.*/}} +{{- if not (hasKey $slice "dnsNames") -}} +{{- $hostName := tuple $service $type $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" -}} +{{- $dnsNames := list $hostName (printf "%s.%s" $hostName $envAll.Release.Namespace) (printf "%s.%s.svc.%s" $hostName $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) -}} +{{- $_ := $dnsNames | set (index $envAll.Values.endpoints $service "host_fqdn_override" "default" "tls") "dnsNames" -}} +{{- end -}} +{{/* Default privateKey size to 4096. This can be overridden. */}} +{{- if not (hasKey $slice "privateKey") -}} +{{- $_ := dict "size" ( printf "%d" 4096 | atoi ) | set (index $envAll.Values.endpoints $service "host_fqdn_override" "default" "tls") "privateKey" -}} +{{- else if empty (index $envAll.Values.endpoints $service "host_fqdn_override" "default" "tls" "privateKey" "size") -}} +{{- $_ := ( printf "%d" 4096 | atoi ) | set (index $envAll.Values.endpoints $service "host_fqdn_override" "default" "tls" "privateKey") "size" -}} +{{- end -}} +{{/* Default duration to 3 months. Note the min is 720h. This can be overridden. */}} +{{- if not (hasKey $slice "duration") -}} +{{- $_ := printf "%s" "2190h" | set (index $envAll.Values.endpoints $service "host_fqdn_override" "default" "tls") "duration" -}} +{{- end -}} +{{/* Default renewBefore to 15 days. This can be overridden. */}} +{{- if not (hasKey $slice "renewBefore") -}} +{{- $_ := printf "%s" "360h" | set (index $envAll.Values.endpoints $service "host_fqdn_override" "default" "tls") "renewBefore" -}} +{{- end -}} +{{/* Default the usage to server auth and client auth. This can be overridden. */}} +{{- if not (hasKey $slice "usages") -}} +{{- $_ := (list "server auth" "client auth") | set (index $envAll.Values.endpoints $service "host_fqdn_override" "default" "tls") "usages" -}} +{{- end -}} +--- +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ index $envAll.Values.endpoints $service "host_fqdn_override" "default" "tls" "secretName" }} + namespace: {{ $envAll.Release.Namespace }} +spec: +{{ $slice | toYaml | indent 2 }} +{{- end -}} diff --git a/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl b/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl new file mode 100644 index 0000000000..332ca99434 --- /dev/null +++ b/helm-toolkit/templates/manifests/_configmap-oslo-policy.tpl @@ -0,0 +1,51 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{/* +abstract: | + Renders out the configmap -oslo-policy. +values: | + conf: + policy.d: + file1: + foo: bar + file2: + foo: baz +usage: | +{{- include "helm-toolkit.manifests.configmap_oslo_policy" (dict "envAll" $envAll "serviceName" "keystone") }} +return: | + --- + apiVersion: v1 + kind: Secret + metadata: + name: keystone-oslo-policy + data: + file1: base64of(foo: bar) + file2: base64of(foo: baz) +*/}} +{{- define "helm-toolkit.manifests.configmap_oslo_policy" -}} +{{- $envAll := index . "envAll" -}} +{{- $serviceName := index . "serviceName" -}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $serviceName }}-oslo-policy +type: Opaque +data: + {{- range $key, $value := index $envAll.Values.conf "policy.d" }} + {{- if $value }} + {{ $key }}: {{ toYaml $value | b64enc }} + {{- else }} + {{ $key }}: {{ "\n" | b64enc }} + {{- end }} + {{- end }} +{{- end -}} diff --git a/helm-toolkit/templates/manifests/_ingress.tpl b/helm-toolkit/templates/manifests/_ingress.tpl new file mode 100644 index 0000000000..792571cb78 --- /dev/null +++ b/helm-toolkit/templates/manifests/_ingress.tpl @@ -0,0 +1,733 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Creates a manifest for a services ingress rules. +examples: + - values: | + network: + api: + ingress: + public: true + classes: + namespace: "nginx" + cluster: "nginx-cluster" + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / + secrets: + tls: + key_manager: + api: + public: barbican-tls-public + endpoints: + cluster_domain_suffix: cluster.local + key_manager: + name: barbican + hosts: + default: barbican-api + public: barbican + host_fqdn_override: + default: null + public: + host: barbican.openstackhelm.example + tls: + crt: | + FOO-CRT + key: | + FOO-KEY + ca: | + FOO-CA_CRT + path: + default: / + scheme: + default: http + public: https + port: + api: + default: 9311 + public: 80 + usage: | + {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "pathType" "Prefix" ) -}} + return: | + --- + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: barbican + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / + + spec: + ingressClassName: "nginx" + rules: + - host: barbican + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: barbican-api + port: + name: b-api + - host: barbican.default + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: barbican-api + port: + name: b-api + - host: barbican.default.svc.cluster.local + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: barbican-api + port: + name: b-api + --- + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: barbican-namespace-fqdn + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / + + spec: + ingressClassName: "nginx" + tls: + - secretName: barbican-tls-public + hosts: + - barbican.openstackhelm.example + rules: + - host: barbican.openstackhelm.example + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: barbican-api + port: + name: b-api + --- + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: barbican-cluster-fqdn + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / + + spec: + ingressClassName: "nginx-cluster" + tls: + - secretName: barbican-tls-public + hosts: + - barbican.openstackhelm.example + rules: + - host: barbican.openstackhelm.example + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: barbican-api + port: + name: b-api + - values: | + network: + api: + ingress: + public: true + classes: + namespace: "nginx" + cluster: "nginx-cluster" + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / + secrets: + tls: + key_manager: + api: + public: barbican-tls-public + endpoints: + cluster_domain_suffix: cluster.local + key_manager: + name: barbican + hosts: + default: barbican-api + public: + host: barbican + tls: + crt: | + FOO-CRT + key: | + FOO-KEY + ca: | + FOO-CA_CRT + host_fqdn_override: + default: null + path: + default: / + scheme: + default: http + public: https + port: + api: + default: 9311 + public: 80 + usage: | + {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "pathType" "Prefix" ) -}} + return: | + --- + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: barbican + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / + + spec: + ingressClassName: "nginx" + tls: + - secretName: barbican-tls-public + hosts: + - barbican + - barbican.default + - barbican.default.svc.cluster.local + rules: + - host: barbican + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: barbican-api + port: + name: b-api + - host: barbican.default + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: barbican-api + port: + name: b-api + - host: barbican.default.svc.cluster.local + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: barbican-api + port: + name: b-api + - values: | + cert_issuer_type: issuer + network: + api: + ingress: + public: true + classes: + namespace: "nginx" + cluster: "nginx-cluster" + annotations: + nginx.ingress.kubernetes.io/secure-backends: "true" + nginx.ingress.kubernetes.io/backend-protocol: "https" + secrets: + tls: + key_manager: + api: + public: barbican-tls-public + internal: barbican-tls-api + endpoints: + cluster_domain_suffix: cluster.local + key_manager: + name: barbican + hosts: + default: barbican-api + public: + host: barbican + tls: + crt: | + FOO-CRT + key: | + FOO-KEY + ca: | + FOO-CA_CRT + host_fqdn_override: + default: null + path: + default: / + scheme: + default: http + public: https + port: + api: + default: 9311 + public: 80 + certs: + barbican_tls_api: + secretName: barbican-tls-api + issuerRef: + name: ca-issuer + kind: Issuer + usage: | + {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" "pathType" "Prefix" ) -}} + return: | + --- + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: barbican + annotations: + cert-manager.io/issuer: ca-issuer + certmanager.k8s.io/issuer: ca-issuer + nginx.ingress.kubernetes.io/backend-protocol: https + nginx.ingress.kubernetes.io/secure-backends: "true" + spec: + ingressClassName: "nginx" + tls: + - secretName: barbican-tls-public-certmanager + hosts: + - barbican + - barbican.default + - barbican.default.svc.cluster.local + rules: + - host: barbican + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: barbican-api + port: + name: b-api + - host: barbican.default + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: barbican-api + port: + name: b-api + - host: barbican.default.svc.cluster.local + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: barbican-api + port: + name: b-api + + - values: | + network: + api: + ingress: + public: true + classes: + namespace: "nginx" + cluster: "nginx-cluster" + annotations: + nginx.ingress.kubernetes.io/secure-backends: "true" + nginx.ingress.kubernetes.io/backend-protocol: "https" + secrets: + tls: + key_manager: + api: + public: barbican-tls-public + internal: barbican-tls-api + endpoints: + cluster_domain_suffix: cluster.local + key_manager: + name: barbican + hosts: + default: barbican-api + public: + host: barbican + tls: + crt: | + FOO-CRT + key: | + FOO-KEY + ca: | + FOO-CA_CRT + host_fqdn_override: + default: null + path: + default: / + scheme: + default: http + public: https + port: + api: + default: 9311 + public: 80 + certs: + barbican_tls_api: + secretName: barbican-tls-api + issuerRef: + name: ca-issuer + kind: ClusterIssuer + usage: | + {{- include "helm-toolkit.manifests.ingress" ( dict "envAll" . "backendServiceType" "key-manager" "backendPort" "b-api" "endpoint" "public" "certIssuer" "ca-issuer" "pathType" "Prefix" ) -}} + return: | + --- + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: barbican + annotations: + cert-manager.io/cluster-issuer: ca-issuer + certmanager.k8s.io/cluster-issuer: ca-issuer + nginx.ingress.kubernetes.io/backend-protocol: https + nginx.ingress.kubernetes.io/secure-backends: "true" + spec: + ingressClassName: "nginx" + tls: + - secretName: barbican-tls-public-certmanager + hosts: + - barbican + - barbican.default + - barbican.default.svc.cluster.local + rules: + - host: barbican + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: barbican-api + port: + name: b-api + - host: barbican.default + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: barbican-api + port: + name: b-api + - host: barbican.default.svc.cluster.local + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: barbican-api + port: + name: b-api + # Sample usage for multiple DNS names associated with the same public + # endpoint and certificate + - values: | + endpoints: + cluster_domain_suffix: cluster.local + grafana: + name: grafana + hosts: + default: grafana-dashboard + public: grafana + host_fqdn_override: + public: + host: grafana.openstackhelm.example + tls: + dnsNames: + - grafana-alt.openstackhelm.example + crt: "BASE64 ENCODED CERT" + key: "BASE64 ENCODED KEY" + network: + grafana: + ingress: + classes: + namespace: "nginx" + cluster: "nginx-cluster" + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / + secrets: + tls: + grafana: + grafana: + public: grafana-tls-public + usage: | + {{- $ingressOpts := dict "envAll" . "backendService" "grafana" "backendServiceType" "grafana" "backendPort" "dashboard" "pathType" "Prefix" -}} + {{ $ingressOpts | include "helm-toolkit.manifests.ingress" }} + return: | + --- + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: grafana + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / + + spec: + ingressClassName: "nginx" + rules: + - host: grafana + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: grafana-dashboard + port: + name: dashboard + - host: grafana.default + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: grafana-dashboard + port: + name: dashboard + - host: grafana.default.svc.cluster.local + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: grafana-dashboard + port: + name: dashboard + --- + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: grafana-namespace-fqdn + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / + + spec: + ingressClassName: "nginx" + tls: + - secretName: grafana-tls-public + hosts: + - grafana.openstackhelm.example + - grafana-alt.openstackhelm.example + rules: + - host: grafana.openstackhelm.example + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: grafana-dashboard + port: + name: dashboard + - host: grafana-alt.openstackhelm.example + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: grafana-dashboard + port: + name: dashboard + --- + apiVersion: networking.k8s.io/v1 + kind: Ingress + metadata: + name: grafana-cluster-fqdn + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / + + spec: + ingressClassName: "nginx-cluster" + tls: + - secretName: grafana-tls-public + hosts: + - grafana.openstackhelm.example + - grafana-alt.openstackhelm.example + rules: + - host: grafana.openstackhelm.example + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: grafana-dashboard + port: + name: dashboard + - host: grafana-alt.openstackhelm.example + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: grafana-dashboard + port: + name: dashboard + +*/}} + +{{- define "helm-toolkit.manifests.ingress._host_rules" -}} +{{- $vHost := index . "vHost" -}} +{{- $backendName := index . "backendName" -}} +{{- $backendPort := index . "backendPort" -}} +{{- $pathType := index . "pathType" -}} +- host: {{ $vHost }} + http: + paths: + - path: / + pathType: {{ $pathType }} + backend: + service: + name: {{ $backendName }} + port: +{{- if or (kindIs "int" $backendPort) (regexMatch "^[0-9]{1,5}$" $backendPort) }} + number: {{ $backendPort | int }} +{{- else }} + name: {{ $backendPort | quote }} +{{- end }} +{{- end }} + +{{- define "helm-toolkit.manifests.ingress" -}} +{{- $envAll := index . "envAll" -}} +{{- $backendService := index . "backendService" | default "api" -}} +{{- $backendServiceType := index . "backendServiceType" -}} +{{- $backendPort := index . "backendPort" -}} +{{- $endpoint := index . "endpoint" | default "public" -}} +{{- $pathType := index . "pathType" | default "Prefix" -}} +{{- $certIssuer := index . "certIssuer" | default "" -}} +{{- $ingressName := tuple $backendServiceType $endpoint $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +{{- $backendName := tuple $backendServiceType "internal" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +{{- $hostName := tuple $backendServiceType $endpoint $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +{{- $hostNameFull := tuple $backendServiceType $endpoint $envAll | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }} +{{- $certIssuerType := "cluster-issuer" -}} +{{- if $envAll.Values.cert_issuer_type }} +{{- $certIssuerType = $envAll.Values.cert_issuer_type }} +{{- end }} +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ $ingressName }} + annotations: +{{- if $certIssuer }} + cert-manager.io/{{ $certIssuerType }}: {{ $certIssuer }} + certmanager.k8s.io/{{ $certIssuerType }}: {{ $certIssuer }} +{{- $slice := index $envAll.Values.endpoints $backendServiceType "host_fqdn_override" "default" "tls" -}} +{{- if (hasKey $slice "duration") }} + cert-manager.io/duration: {{ index $slice "duration" }} +{{- end }} +{{- end }} +{{ toYaml (index $envAll.Values.network $backendService "ingress" "annotations") | indent 4 }} +spec: + ingressClassName: {{ index $envAll.Values.network $backendService "ingress" "classes" "namespace" | quote }} +{{- $host := index $envAll.Values.endpoints ( $backendServiceType | replace "-" "_" ) "hosts" }} +{{- if $certIssuer }} +{{- $secretName := index $envAll.Values.secrets "tls" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }} +{{- $_ := required "You need to specify a secret in your values for the endpoint" $secretName }} + tls: + - secretName: {{ printf "%s-ing" $secretName }} + hosts: +{{- range $key1, $vHost := tuple $hostName (printf "%s.%s" $hostName $envAll.Release.Namespace) (printf "%s.%s.svc.%s" $hostName $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) }} + - {{ $vHost }} +{{- end }} +{{- else }} +{{- if hasKey $host $endpoint }} +{{- $endpointHost := index $host $endpoint }} +{{- if kindIs "map" $endpointHost }} +{{- if hasKey $endpointHost "tls" }} +{{- if and ( not ( empty $endpointHost.tls.key ) ) ( not ( empty $endpointHost.tls.crt ) ) }} +{{- $secretName := index $envAll.Values.secrets "tls" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }} +{{- $_ := required "You need to specify a secret in your values for the endpoint" $secretName }} + tls: + - secretName: {{ $secretName }} + hosts: +{{- range $key1, $vHost := tuple $hostName (printf "%s.%s" $hostName $envAll.Release.Namespace) (printf "%s.%s.svc.%s" $hostName $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) }} + - {{ $vHost }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} + rules: +{{- range $key1, $vHost := tuple $hostName (printf "%s.%s" $hostName $envAll.Release.Namespace) (printf "%s.%s.svc.%s" $hostName $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) }} +{{- $hostRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort "pathType" $pathType }} +{{ $hostRules | include "helm-toolkit.manifests.ingress._host_rules" | indent 4 }} +{{- end }} +{{- if not ( hasSuffix ( printf ".%s.svc.%s" $envAll.Release.Namespace $envAll.Values.endpoints.cluster_domain_suffix) $hostNameFull) }} +{{- $ingressConf := $envAll.Values.network -}} +{{- $ingressClasses := ternary (tuple "namespace") (tuple "namespace" "cluster") (and (hasKey $ingressConf "use_external_ingress_controller") $ingressConf.use_external_ingress_controller) }} +{{- range $key2, $ingressController := $ingressClasses }} +{{- $vHosts := list $hostNameFull }} +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: {{ printf "%s-%s-%s" $ingressName $ingressController "fqdn" }} + annotations: +{{ toYaml (index $envAll.Values.network $backendService "ingress" "annotations") | indent 4 }} +spec: + ingressClassName: {{ index $envAll.Values.network $backendService "ingress" "classes" $ingressController | quote }} +{{- $host := index $envAll.Values.endpoints ( $backendServiceType | replace "-" "_" ) "host_fqdn_override" }} +{{- if hasKey $host $endpoint }} +{{- $endpointHost := index $host $endpoint }} +{{- if kindIs "map" $endpointHost }} +{{- if hasKey $endpointHost "tls" }} +{{- range $v := without (index $endpointHost.tls "dnsNames" | default list) $hostNameFull }} +{{- $vHosts = append $vHosts $v }} +{{- end }} +{{- if hasKey $envAll.Values.endpoints "alias_fqdn" }} +{{- $alias_host := $envAll.Values.endpoints.alias_fqdn }} +{{- $vHosts = append $vHosts $alias_host }} +{{- end }} +{{- $secretName := index $envAll.Values.secrets "tls" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }} +{{- $_ := required "You need to specify a secret in your values for the endpoint" $secretName }} + tls: + - secretName: {{ $secretName }} + hosts: +{{- range $vHost := $vHosts }} + - {{ $vHost }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} + rules: +{{- range $vHost := $vHosts }} +{{- $hostNameFullRules := dict "vHost" $vHost "backendName" $backendName "backendPort" $backendPort "pathType" $pathType }} +{{ $hostNameFullRules | include "helm-toolkit.manifests.ingress._host_rules" | indent 4 }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/helm-toolkit/templates/manifests/_job-bootstrap.tpl b/helm-toolkit/templates/manifests/_job-bootstrap.tpl new file mode 100644 index 0000000000..6b77004f0d --- /dev/null +++ b/helm-toolkit/templates/manifests/_job-bootstrap.tpl @@ -0,0 +1,142 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# This function creates a manifest for db creation and user management. +# It can be used in charts dict created similar to the following: +# {- $bootstrapJob := dict "envAll" . "serviceName" "senlin" -} +# { $bootstrapJob | include "helm-toolkit.manifests.job_bootstrap" } + +{{- define "helm-toolkit.manifests.job_bootstrap" -}} +{{- $envAll := index . "envAll" -}} +{{- $serviceName := index . "serviceName" -}} +{{- $jobAnnotations := index . "jobAnnotations" -}} +{{- $jobLabels := index . "jobLabels" -}} +{{- $nodeSelector := index . "nodeSelector" | default ( dict $envAll.Values.labels.job.node_selector_key $envAll.Values.labels.job.node_selector_value ) -}} +{{- $tolerationsEnabled := index . "tolerationsEnabled" | default false -}} +{{- $podVolMounts := index . "podVolMounts" | default false -}} +{{- $podVols := index . "podVols" | default false -}} +{{- $configMapBin := index . "configMapBin" | default (printf "%s-%s" $serviceName "bin" ) -}} +{{- $configMapEtc := index . "configMapEtc" | default (printf "%s-%s" $serviceName "etc" ) -}} +{{- $configFile := index . "configFile" | default (printf "/etc/%s/%s.conf" $serviceName $serviceName ) -}} +{{- $logConfigFile := index . "logConfigFile" | default (printf "/etc/%s/logging.conf" $serviceName ) -}} +{{- $tlsSecret := index . "tlsSecret" | default "" -}} +{{- $keystoneUser := index . "keystoneUser" | default $serviceName -}} +{{- $openrc := index . "openrc" | default "true" -}} +{{- $secretBin := index . "secretBin" -}} +{{- $backoffLimit := index . "backoffLimit" | default "1000" -}} +{{- $activeDeadlineSeconds := index . "activeDeadlineSeconds" -}} +{{- $serviceNamePretty := $serviceName | replace "_" "-" -}} + +{{- $serviceAccountName := printf "%s-%s" $serviceNamePretty "bootstrap" }} +{{ tuple $envAll "bootstrap" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ printf "%s-%s" $serviceNamePretty "bootstrap" | quote }} + labels: +{{ tuple $envAll $serviceName "bootstrap" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- if $jobLabels }} +{{ toYaml $jobLabels | indent 4 }} +{{- end }} + annotations: +{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}} +{{- if $jobAnnotations }} +{{ toYaml $jobAnnotations | indent 4 }} +{{- end }} +spec: + backoffLimit: {{ $backoffLimit }} +{{- if $activeDeadlineSeconds }} + activeDeadlineSeconds: {{ $activeDeadlineSeconds }} +{{- end }} + template: + metadata: + labels: +{{ tuple $envAll $serviceName "bootstrap" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} +{{- if $jobLabels }} +{{ toYaml $jobLabels | indent 8 }} +{{- end }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + spec: + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + {{ tuple $envAll "bootstrap" | include "helm-toolkit.snippets.kubernetes_image_pull_secrets" | indent 6 }} + nodeSelector: +{{ toYaml $nodeSelector | indent 8 }} +{{- if $tolerationsEnabled }} +{{ tuple $envAll $serviceName | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }} +{{- end}} + initContainers: +{{ tuple $envAll "bootstrap" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: bootstrap + image: {{ $envAll.Values.images.tags.bootstrap }} + imagePullPolicy: {{ $envAll.Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.bootstrap | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{- if eq $openrc "true" }} + env: +{{- with $env := dict "ksUserSecret" ( index $envAll.Values.secrets.identity $keystoneUser ) "useCA" (ne $tlsSecret "") }} +{{- include "helm-toolkit.snippets.keystone_openrc_env_vars" $env | indent 12 }} +{{- end }} +{{- end }} + command: + - /bin/bash + - -c + - /tmp/bootstrap.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: bootstrap-sh + mountPath: /tmp/bootstrap.sh + subPath: bootstrap.sh + readOnly: true + - name: etc-service + mountPath: {{ dir $configFile | quote }} + - name: bootstrap-conf + mountPath: {{ $configFile | quote }} + subPath: {{ base $configFile | quote }} + readOnly: true + - name: bootstrap-conf + mountPath: {{ $logConfigFile | quote }} + subPath: {{ base $logConfigFile | quote }} + readOnly: true +{{ dict "enabled" (ne $tlsSecret "") "name" $tlsSecret | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }} +{{- if $podVolMounts }} +{{ $podVolMounts | toYaml | indent 12 }} +{{- end }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: bootstrap-sh +{{- if $secretBin }} + secret: + secretName: {{ $secretBin | quote }} + defaultMode: 0555 +{{- else }} + configMap: + name: {{ $configMapBin | quote }} + defaultMode: 0555 +{{- end }} + - name: etc-service + emptyDir: {} + - name: bootstrap-conf + secret: + secretName: {{ $configMapEtc | quote }} + defaultMode: 0444 +{{- dict "enabled" (ne $tlsSecret "") "name" $tlsSecret | include "helm-toolkit.snippets.tls_volume" | indent 8 }} +{{- if $podVols }} +{{ $podVols | toYaml | indent 8 }} +{{- end }} +{{- end }} diff --git a/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl b/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl new file mode 100644 index 0000000000..2b7ff2cdcb --- /dev/null +++ b/helm-toolkit/templates/manifests/_job-db-drop-mysql.tpl @@ -0,0 +1,171 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# This function creates a manifest for db creation and user management. +# It can be used in charts dict created similar to the following: +# {- $dbToDropJob := dict "envAll" . "serviceName" "senlin" -} +# { $dbToDropJob | include "helm-toolkit.manifests.job_db_drop_mysql" } +# +# If the service does not use oslo then the db can be managed with: +# {- $dbToDrop := dict "inputType" "secret" "adminSecret" .Values.secrets.oslo_db.admin "userSecret" .Values.secrets.oslo_db.horizon -} +# {- $dbToDropJob := dict "envAll" . "serviceName" "horizon" "dbToDrop" $dbToDrop -} +# { $dbToDropJob | include "helm-toolkit.manifests.job_db_drop_mysql" } + +{{- define "helm-toolkit.manifests.job_db_drop_mysql" -}} +{{- $envAll := index . "envAll" -}} +{{- $serviceName := index . "serviceName" -}} +{{- $jobAnnotations := index . "jobAnnotations" -}} +{{- $jobLabels := index . "jobLabels" -}} +{{- $nodeSelector := index . "nodeSelector" | default ( dict $envAll.Values.labels.job.node_selector_key $envAll.Values.labels.job.node_selector_value ) -}} +{{- $tolerationsEnabled := index . "tolerationsEnabled" | default false -}} +{{- $configMapBin := index . "configMapBin" | default (printf "%s-%s" $serviceName "bin" ) -}} +{{- $configMapEtc := index . "configMapEtc" | default (printf "%s-%s" $serviceName "etc" ) -}} +{{- $dbToDrop := index . "dbToDrop" | default ( dict "adminSecret" $envAll.Values.secrets.oslo_db.admin "configFile" (printf "/etc/%s/%s.conf" $serviceName $serviceName ) "logConfigFile" (printf "/etc/%s/logging.conf" $serviceName ) "configDbSection" "database" "configDbKey" "connection" ) -}} +{{- $dbsToDrop := default (list $dbToDrop) (index . "dbsToDrop") }} +{{- $secretBin := index . "secretBin" -}} +{{- $backoffLimit := index . "backoffLimit" | default "1000" -}} +{{- $activeDeadlineSeconds := index . "activeDeadlineSeconds" -}} +{{- $serviceNamePretty := $serviceName | replace "_" "-" -}} +{{- $dbAdminTlsSecret := index . "dbAdminTlsSecret" | default "" -}} + +{{- $serviceAccountName := printf "%s-%s" $serviceNamePretty "db-drop" }} +{{ tuple $envAll "db_drop" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ printf "%s-%s" $serviceNamePretty "db-drop" | quote }} + labels: +{{ tuple $envAll $serviceName "db-drop" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- if $jobLabels }} +{{ toYaml $jobLabels | indent 4 }} +{{- end }} + annotations: + "helm.sh/hook": pre-delete + "helm.sh/hook-delete-policy": hook-succeeded +{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}} +{{- if $jobAnnotations }} +{{ toYaml $jobAnnotations | indent 4 }} +{{- end }} +spec: + backoffLimit: {{ $backoffLimit }} +{{- if $activeDeadlineSeconds }} + activeDeadlineSeconds: {{ $activeDeadlineSeconds }} +{{- end }} + template: + metadata: + labels: +{{ tuple $envAll $serviceName "db-drop" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} +{{- if $jobLabels }} +{{ toYaml $jobLabels | indent 8 }} +{{- end }} + spec: + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + {{ tuple $envAll "db_drop" | include "helm-toolkit.snippets.kubernetes_image_pull_secrets" | indent 6 }} + nodeSelector: +{{ toYaml $nodeSelector | indent 8 }} +{{- if $tolerationsEnabled }} +{{ tuple $envAll $serviceName | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }} +{{- end}} + initContainers: +{{ tuple $envAll "db_drop" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: +{{- range $key1, $dbToDrop := $dbsToDrop }} +{{ $dbToDropType := default "oslo" $dbToDrop.inputType }} + - name: {{ printf "%s-%s-%d" $serviceNamePretty "db-drop" $key1 | quote }} + image: {{ $envAll.Values.images.tags.db_drop }} + imagePullPolicy: {{ $envAll.Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.db_drop | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + env: + - name: ROOT_DB_CONNECTION + valueFrom: + secretKeyRef: + name: {{ $dbToDrop.adminSecret | quote }} + key: DB_CONNECTION +{{- if eq $dbToDropType "oslo" }} + - name: OPENSTACK_CONFIG_FILE + value: {{ $dbToDrop.configFile | quote }} + - name: OPENSTACK_CONFIG_DB_SECTION + value: {{ $dbToDrop.configDbSection | quote }} + - name: OPENSTACK_CONFIG_DB_KEY + value: {{ $dbToDrop.configDbKey | quote }} +{{- end }} +{{- if $envAll.Values.manifests.certificates }} + - name: MARIADB_X509 + value: "REQUIRE X509" +{{- end }} +{{- if eq $dbToDropType "secret" }} + - name: DB_CONNECTION + valueFrom: + secretKeyRef: + name: {{ $dbToDrop.userSecret | quote }} + key: DB_CONNECTION +{{- end }} + command: + - /tmp/db-drop.py + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: db-drop-sh + mountPath: /tmp/db-drop.py + subPath: db-drop.py + readOnly: true + +{{- if eq $dbToDropType "oslo" }} + - name: etc-service + mountPath: {{ dir $dbToDrop.configFile | quote }} + - name: db-drop-conf + mountPath: {{ $dbToDrop.configFile | quote }} + subPath: {{ base $dbToDrop.configFile | quote }} + readOnly: true + - name: db-drop-conf + mountPath: {{ $dbToDrop.logConfigFile | quote }} + subPath: {{ base $dbToDrop.logConfigFile | quote }} + readOnly: true +{{- end }} +{{- if $envAll.Values.manifests.certificates }} +{{- dict "enabled" $envAll.Values.manifests.certificates "name" $dbAdminTlsSecret "path" "/etc/mysql/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }} +{{- end }} +{{- end }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: db-drop-sh +{{- if $secretBin }} + secret: + secretName: {{ $secretBin | quote }} + defaultMode: 0555 +{{- else }} + configMap: + name: {{ $configMapBin | quote }} + defaultMode: 0555 +{{- end }} +{{- if $envAll.Values.manifests.certificates }} +{{- dict "enabled" $envAll.Values.manifests.certificates "name" $dbAdminTlsSecret | include "helm-toolkit.snippets.tls_volume" | indent 8 }} +{{- end }} +{{- $local := dict "configMapBinFirst" true -}} +{{- range $key1, $dbToDrop := $dbsToDrop }} +{{- $dbToDropType := default "oslo" $dbToDrop.inputType }} +{{- if and (eq $dbToDropType "oslo") $local.configMapBinFirst }} +{{- $_ := set $local "configMapBinFirst" false }} + - name: etc-service + emptyDir: {} + - name: db-drop-conf + secret: + secretName: {{ $configMapEtc | quote }} + defaultMode: 0444 +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl b/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl new file mode 100644 index 0000000000..b8a1dce3b3 --- /dev/null +++ b/helm-toolkit/templates/manifests/_job-db-init-mysql.tpl @@ -0,0 +1,170 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# This function creates a manifest for db creation and user management. +# It can be used in charts dict created similar to the following: +# {- $dbToInitJob := dict "envAll" . "serviceName" "senlin" -} +# { $dbToInitJob | include "helm-toolkit.manifests.job_db_init_mysql" } +# +# If the service does not use oslo then the db can be managed with: +# {- $dbToInit := dict "inputType" "secret" "adminSecret" .Values.secrets.oslo_db.admin "userSecret" .Values.secrets.oslo_db.horizon -} +# {- $dbToInitJob := dict "envAll" . "serviceName" "horizon" "dbToInit" $dbToInit -} +# { $dbToInitJob | include "helm-toolkit.manifests.job_db_init_mysql" } + +{{- define "helm-toolkit.manifests.job_db_init_mysql" -}} +{{- $envAll := index . "envAll" -}} +{{- $serviceName := index . "serviceName" -}} +{{- $jobAnnotations := index . "jobAnnotations" -}} +{{- $jobLabels := index . "jobLabels" -}} +{{- $nodeSelector := index . "nodeSelector" | default ( dict $envAll.Values.labels.job.node_selector_key $envAll.Values.labels.job.node_selector_value ) -}} +{{- $tolerationsEnabled := index . "tolerationsEnabled" | default false -}} +{{- $configMapBin := index . "configMapBin" | default (printf "%s-%s" $serviceName "bin" ) -}} +{{- $configMapEtc := index . "configMapEtc" | default (printf "%s-%s" $serviceName "etc" ) -}} +{{- $dbToInit := index . "dbToInit" | default ( dict "adminSecret" $envAll.Values.secrets.oslo_db.admin "configFile" (printf "/etc/%s/%s.conf" $serviceName $serviceName ) "logConfigFile" (printf "/etc/%s/logging.conf" $serviceName ) "configDbSection" "database" "configDbKey" "connection" ) -}} +{{- $dbsToInit := default (list $dbToInit) (index . "dbsToInit") }} +{{- $secretBin := index . "secretBin" -}} +{{- $backoffLimit := index . "backoffLimit" | default "1000" -}} +{{- $activeDeadlineSeconds := index . "activeDeadlineSeconds" -}} +{{- $serviceNamePretty := $serviceName | replace "_" "-" -}} +{{- $dbAdminTlsSecret := index . "dbAdminTlsSecret" | default "" -}} + +{{- $serviceAccountName := printf "%s-%s" $serviceNamePretty "db-init" }} +{{ tuple $envAll "db_init" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ printf "%s-%s" $serviceNamePretty "db-init" | quote }} + labels: +{{ tuple $envAll $serviceName "db-init" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- if $jobLabels }} +{{ toYaml $jobLabels | indent 4 }} +{{- end }} + annotations: +{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}} +{{- if $jobAnnotations }} +{{ toYaml $jobAnnotations | indent 4 }} +{{- end }} +spec: + backoffLimit: {{ $backoffLimit }} +{{- if $activeDeadlineSeconds }} + activeDeadlineSeconds: {{ $activeDeadlineSeconds }} +{{- end }} + template: + metadata: + labels: +{{ tuple $envAll $serviceName "db-init" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} +{{- if $jobLabels }} +{{ toYaml $jobLabels | indent 8 }} +{{- end }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + spec: + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + {{ tuple $envAll "db_init" | include "helm-toolkit.snippets.kubernetes_image_pull_secrets" | indent 6 }} + nodeSelector: +{{ toYaml $nodeSelector | indent 8 }} +{{- if $tolerationsEnabled }} +{{ tuple $envAll $serviceName | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }} +{{- end}} + initContainers: +{{ tuple $envAll "db_init" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: +{{- range $key1, $dbToInit := $dbsToInit }} +{{ $dbToInitType := default "oslo" $dbToInit.inputType }} + - name: {{ printf "%s-%s-%d" $serviceNamePretty "db-init" $key1 | quote }} + image: {{ $envAll.Values.images.tags.db_init }} + imagePullPolicy: {{ $envAll.Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.db_init | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + env: + - name: ROOT_DB_CONNECTION + valueFrom: + secretKeyRef: + name: {{ $dbToInit.adminSecret | quote }} + key: DB_CONNECTION +{{- if eq $dbToInitType "oslo" }} + - name: OPENSTACK_CONFIG_FILE + value: {{ $dbToInit.configFile | quote }} + - name: OPENSTACK_CONFIG_DB_SECTION + value: {{ $dbToInit.configDbSection | quote }} + - name: OPENSTACK_CONFIG_DB_KEY + value: {{ $dbToInit.configDbKey | quote }} +{{- end }} +{{- if eq $dbToInitType "secret" }} + - name: DB_CONNECTION + valueFrom: + secretKeyRef: + name: {{ $dbToInit.userSecret | quote }} + key: DB_CONNECTION +{{- end }} +{{- if $envAll.Values.manifests.certificates }} + - name: MARIADB_X509 + value: "REQUIRE X509" +{{- end }} + command: + - /tmp/db-init.py + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: db-init-sh + mountPath: /tmp/db-init.py + subPath: db-init.py + readOnly: true +{{- if eq $dbToInitType "oslo" }} + - name: etc-service + mountPath: {{ dir $dbToInit.configFile | quote }} + - name: db-init-conf + mountPath: {{ $dbToInit.configFile | quote }} + subPath: {{ base $dbToInit.configFile | quote }} + readOnly: true + - name: db-init-conf + mountPath: {{ $dbToInit.logConfigFile | quote }} + subPath: {{ base $dbToInit.logConfigFile | quote }} + readOnly: true +{{- end }} +{{- if $envAll.Values.manifests.certificates }} +{{- dict "enabled" $envAll.Values.manifests.certificates "name" $dbAdminTlsSecret "path" "/etc/mysql/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }} +{{- end }} +{{- end }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: db-init-sh +{{- if $secretBin }} + secret: + secretName: {{ $secretBin | quote }} + defaultMode: 0555 +{{- else }} + configMap: + name: {{ $configMapBin | quote }} + defaultMode: 0555 +{{- end }} +{{- if $envAll.Values.manifests.certificates }} +{{- dict "enabled" $envAll.Values.manifests.certificates "name" $dbAdminTlsSecret | include "helm-toolkit.snippets.tls_volume" | indent 8 }} +{{- end }} +{{- $local := dict "configMapBinFirst" true -}} +{{- range $key1, $dbToInit := $dbsToInit }} +{{- $dbToInitType := default "oslo" $dbToInit.inputType }} +{{- if and (eq $dbToInitType "oslo") $local.configMapBinFirst }} +{{- $_ := set $local "configMapBinFirst" false }} + - name: etc-service + emptyDir: {} + - name: db-init-conf + secret: + secretName: {{ $configMapEtc | quote }} + defaultMode: 0444 +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/helm-toolkit/templates/manifests/_job-db-sync.tpl b/helm-toolkit/templates/manifests/_job-db-sync.tpl new file mode 100644 index 0000000000..03b048416a --- /dev/null +++ b/helm-toolkit/templates/manifests/_job-db-sync.tpl @@ -0,0 +1,140 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# This function creates a manifest for db migration and management. +# It can be used in charts dict created similar to the following: +# {- $dbSyncJob := dict "envAll" . "serviceName" "senlin" -} +# { $dbSyncJob | include "helm-toolkit.manifests.job_db_sync" } + +{{- define "helm-toolkit.manifests.job_db_sync" -}} +{{- $envAll := index . "envAll" -}} +{{- $serviceName := index . "serviceName" -}} +{{- $jobNameRef := printf "%s_%s" $serviceName "db_sync" -}} +{{- $jobAnnotations := index . "jobAnnotations" -}} +{{- $jobLabels := index . "jobLabels" -}} +{{- $nodeSelector := index . "nodeSelector" | default ( dict $envAll.Values.labels.job.node_selector_key $envAll.Values.labels.job.node_selector_value ) -}} +{{- $tolerationsEnabled := index . "tolerationsEnabled" | default false -}} +{{- $configMapBin := index . "configMapBin" | default (printf "%s-%s" $serviceName "bin" ) -}} +{{- $configMapEtc := index . "configMapEtc" | default (printf "%s-%s" $serviceName "etc" ) -}} +{{- $podMount := index (index $envAll.Values.pod.mounts $jobNameRef | default dict) $jobNameRef | default dict -}} +{{- $podVolMounts := (concat ((index $podMount "volumeMounts" | default list)) ((index . "podVolMounts") | default (list))) | uniq -}} +{{- $podVols := (concat ((index $podMount "volumes" | default list)) ((index . "podVols") | default (list))) | uniq -}} +{{- $podEnvVars := index . "podEnvVars" | default false -}} +{{- $dbToSync := index . "dbToSync" | default ( dict "configFile" (printf "/etc/%s/%s.conf" $serviceName $serviceName ) "logConfigFile" (printf "/etc/%s/logging.conf" $serviceName ) "image" ( index $envAll.Values.images.tags ( printf "%s_db_sync" $serviceName )) ) -}} +{{- $secretBin := index . "secretBin" -}} +{{- $backoffLimit := index . "backoffLimit" | default "1000" -}} +{{- $activeDeadlineSeconds := index . "activeDeadlineSeconds" -}} +{{- $serviceNamePretty := $serviceName | replace "_" "-" -}} +{{- $dbAdminTlsSecret := index . "dbAdminTlsSecret" | default "" -}} + +{{- $serviceAccountName := printf "%s-%s" $serviceNamePretty "db-sync" }} +{{ tuple $envAll "db_sync" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ printf "%s-%s" $serviceNamePretty "db-sync" | quote }} + labels: +{{ tuple $envAll $serviceName "db-sync" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- if $jobLabels }} +{{ toYaml $jobLabels | indent 4 }} +{{- end }} + annotations: +{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}} +{{- if $jobAnnotations }} +{{ toYaml $jobAnnotations | indent 4 }} +{{- end }} +spec: + backoffLimit: {{ $backoffLimit }} +{{- if $activeDeadlineSeconds }} + activeDeadlineSeconds: {{ $activeDeadlineSeconds }} +{{- end }} + template: + metadata: + labels: +{{ tuple $envAll $serviceName "db-sync" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} +{{- if $jobLabels }} +{{ toYaml $jobLabels | indent 8 }} +{{- end }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + spec: + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + {{ tuple $envAll "db_sync" | include "helm-toolkit.snippets.kubernetes_image_pull_secrets" | indent 6 }} + nodeSelector: +{{ toYaml $nodeSelector | indent 8 }} +{{- if $tolerationsEnabled }} +{{ tuple $envAll $serviceName | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }} +{{- end}} + initContainers: +{{ tuple $envAll "db_sync" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: {{ printf "%s-%s" $serviceNamePretty "db-sync" | quote }} + image: {{ $dbToSync.image | quote }} + imagePullPolicy: {{ $envAll.Values.images.pull_policy | quote }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.db_sync | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{- if $podEnvVars }} + env: +{{ $podEnvVars | toYaml | indent 12 }} +{{- end }} + command: + - /bin/bash + - -c + - /tmp/db-sync.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: db-sync-sh + mountPath: /tmp/db-sync.sh + subPath: db-sync.sh + readOnly: true + - name: etc-service + mountPath: {{ dir $dbToSync.configFile | quote }} + - name: db-sync-conf + mountPath: {{ $dbToSync.configFile | quote }} + subPath: {{ base $dbToSync.configFile | quote }} + readOnly: true + - name: db-sync-conf + mountPath: {{ $dbToSync.logConfigFile | quote }} + subPath: {{ base $dbToSync.logConfigFile | quote }} + readOnly: true +{{- dict "enabled" $envAll.Values.manifests.certificates "name" $dbAdminTlsSecret "path" "/etc/mysql/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }} +{{- if $podVolMounts }} +{{ $podVolMounts | toYaml | indent 12 }} +{{- end }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: db-sync-sh +{{- if $secretBin }} + secret: + secretName: {{ $secretBin | quote }} + defaultMode: 0555 +{{- else }} + configMap: + name: {{ $configMapBin | quote }} + defaultMode: 0555 +{{- end }} + - name: etc-service + emptyDir: {} + - name: db-sync-conf + secret: + secretName: {{ $configMapEtc | quote }} + defaultMode: 0444 +{{- dict "enabled" $envAll.Values.manifests.certificates "name" $dbAdminTlsSecret | include "helm-toolkit.snippets.tls_volume" | indent 8 }} +{{- if $podVols }} +{{ $podVols | toYaml | indent 8 }} +{{- end }} +{{- end }} diff --git a/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl b/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl new file mode 100644 index 0000000000..d69c9e6ec1 --- /dev/null +++ b/helm-toolkit/templates/manifests/_job-ks-endpoints.tpl @@ -0,0 +1,131 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# This function creates a manifest for keystone service management. +# It can be used in charts dict created similar to the following: +# {- $ksEndpointJob := dict "envAll" . "serviceName" "senlin" "serviceTypes" ( tuple "clustering" ) -} +# { $ksEndpointJob | include "helm-toolkit.manifests.job_ks_endpoints" } + +{{- define "helm-toolkit.manifests.job_ks_endpoints" -}} +{{- $envAll := index . "envAll" -}} +{{- $serviceName := index . "serviceName" -}} +{{- $serviceTypes := index . "serviceTypes" -}} +{{- $jobAnnotations := index . "jobAnnotations" -}} +{{- $jobLabels := index . "jobLabels" -}} +{{- $nodeSelector := index . "nodeSelector" | default ( dict $envAll.Values.labels.job.node_selector_key $envAll.Values.labels.job.node_selector_value ) -}} +{{- $tolerationsEnabled := index . "tolerationsEnabled" | default false -}} +{{- $configMapBin := index . "configMapBin" | default (printf "%s-%s" $serviceName "bin" ) -}} +{{- $secretBin := index . "secretBin" -}} +{{- $tlsSecret := index . "tlsSecret" | default "" -}} +{{- $backoffLimit := index . "backoffLimit" | default "1000" -}} +{{- $activeDeadlineSeconds := index . "activeDeadlineSeconds" -}} +{{- $serviceNamePretty := $serviceName | replace "_" "-" -}} +{{- $restartPolicy_ := "OnFailure" -}} +{{- if hasKey $envAll.Values "jobs" -}} +{{- if hasKey $envAll.Values.jobs "ks_endpoints" -}} +{{- $restartPolicy_ = $envAll.Values.jobs.ks_endpoints.restartPolicy | default $restartPolicy_ }} +{{- end }} +{{- end }} +{{- $restartPolicy := index . "restartPolicy" | default $restartPolicy_ -}} + +{{- $serviceAccountName := printf "%s-%s" $serviceNamePretty "ks-endpoints" }} +{{ tuple $envAll "ks_endpoints" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ printf "%s-%s" $serviceNamePretty "ks-endpoints" | quote }} + labels: +{{ tuple $envAll $serviceName "ks-endpoints" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- if $jobLabels }} +{{ toYaml $jobLabels | indent 4 }} +{{- end }} + annotations: +{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}} +{{- if $jobAnnotations }} +{{ toYaml $jobAnnotations | indent 4 }} +{{- end }} +spec: + backoffLimit: {{ $backoffLimit }} +{{- if $activeDeadlineSeconds }} + activeDeadlineSeconds: {{ $activeDeadlineSeconds }} +{{- end }} + template: + metadata: + labels: +{{ tuple $envAll $serviceName "ks-endpoints" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} +{{- if $jobLabels }} +{{ toYaml $jobLabels | indent 8 }} +{{- end }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + spec: + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: {{ $restartPolicy }} + {{ tuple $envAll "ks_endpoints" | include "helm-toolkit.snippets.kubernetes_image_pull_secrets" | indent 6 }} + nodeSelector: +{{ toYaml $nodeSelector | indent 8 }} +{{- if $tolerationsEnabled }} +{{ tuple $envAll $serviceName | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }} +{{- end}} + initContainers: +{{ tuple $envAll "ks_endpoints" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: +{{- range $key1, $osServiceType := $serviceTypes }} +{{- range $key2, $osServiceEndPoint := tuple "admin" "internal" "public" }} + - name: {{ printf "%s-%s-%s" $osServiceType "ks-endpoints" $osServiceEndPoint | quote }} + image: {{ $envAll.Values.images.tags.ks_endpoints }} + imagePullPolicy: {{ $envAll.Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.ks_endpoints | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + command: + - /bin/bash + - -c + - /tmp/ks-endpoints.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: ks-endpoints-sh + mountPath: /tmp/ks-endpoints.sh + subPath: ks-endpoints.sh + readOnly: true +{{ dict "enabled" true "name" $tlsSecret "ca" true | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }} + env: +{{- with $env := dict "ksUserSecret" $envAll.Values.secrets.identity.admin "useCA" (ne $tlsSecret "") }} +{{- include "helm-toolkit.snippets.keystone_openrc_env_vars" $env | indent 12 }} +{{- end }} + - name: OS_SVC_ENDPOINT + value: {{ $osServiceEndPoint | quote }} + - name: OS_SERVICE_NAME + value: {{ tuple $osServiceType $envAll | include "helm-toolkit.endpoints.keystone_endpoint_name_lookup" }} + - name: OS_SERVICE_TYPE + value: {{ $osServiceType | quote }} + - name: OS_SERVICE_ENDPOINT + value: {{ tuple $osServiceType $osServiceEndPoint "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | quote }} +{{- end }} +{{- end }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: ks-endpoints-sh +{{- if $secretBin }} + secret: + secretName: {{ $secretBin | quote }} + defaultMode: 0555 +{{- else }} + configMap: + name: {{ $configMapBin | quote }} + defaultMode: 0555 +{{- end }} +{{- dict "enabled" true "name" $tlsSecret | include "helm-toolkit.snippets.tls_volume" | indent 8 }} +{{- end }} diff --git a/helm-toolkit/templates/manifests/_job-ks-service.tpl b/helm-toolkit/templates/manifests/_job-ks-service.tpl new file mode 100644 index 0000000000..9604c63728 --- /dev/null +++ b/helm-toolkit/templates/manifests/_job-ks-service.tpl @@ -0,0 +1,125 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# This function creates a manifest for keystone service management. +# It can be used in charts dict created similar to the following: +# {- $ksServiceJob := dict "envAll" . "serviceName" "senlin" "serviceTypes" ( tuple "clustering" ) -} +# { $ksServiceJob | include "helm-toolkit.manifests.job_ks_service" } + +{{- define "helm-toolkit.manifests.job_ks_service" -}} +{{- $envAll := index . "envAll" -}} +{{- $serviceName := index . "serviceName" -}} +{{- $serviceTypes := index . "serviceTypes" -}} +{{- $jobAnnotations := index . "jobAnnotations" -}} +{{- $jobLabels := index . "jobLabels" -}} +{{- $nodeSelector := index . "nodeSelector" | default ( dict $envAll.Values.labels.job.node_selector_key $envAll.Values.labels.job.node_selector_value ) -}} +{{- $tolerationsEnabled := index . "tolerationsEnabled" | default false -}} +{{- $configMapBin := index . "configMapBin" | default (printf "%s-%s" $serviceName "bin" ) -}} +{{- $secretBin := index . "secretBin" -}} +{{- $tlsSecret := index . "tlsSecret" | default "" -}} +{{- $backoffLimit := index . "backoffLimit" | default "1000" -}} +{{- $activeDeadlineSeconds := index . "activeDeadlineSeconds" -}} +{{- $serviceNamePretty := $serviceName | replace "_" "-" -}} +{{- $restartPolicy_ := "OnFailure" -}} +{{- if hasKey $envAll.Values "jobs" -}} +{{- if hasKey $envAll.Values.jobs "ks_service" -}} +{{- $restartPolicy_ = $envAll.Values.jobs.ks_service.restartPolicy | default $restartPolicy_ }} +{{- end }} +{{- end }} +{{- $restartPolicy := index . "restartPolicy" | default $restartPolicy_ -}} + +{{- $serviceAccountName := printf "%s-%s" $serviceNamePretty "ks-service" }} +{{ tuple $envAll "ks_service" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ printf "%s-%s" $serviceNamePretty "ks-service" | quote }} + labels: +{{ tuple $envAll $serviceName "ks-service" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- if $jobLabels }} +{{ toYaml $jobLabels | indent 4 }} +{{- end }} + annotations: +{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}} +{{- if $jobAnnotations }} +{{ toYaml $jobAnnotations | indent 4 }} +{{- end }} +spec: + backoffLimit: {{ $backoffLimit }} +{{- if $activeDeadlineSeconds }} + activeDeadlineSeconds: {{ $activeDeadlineSeconds }} +{{- end }} + template: + metadata: + labels: +{{ tuple $envAll $serviceName "ks-service" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} +{{- if $jobLabels }} +{{ toYaml $jobLabels | indent 8 }} +{{- end }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + spec: + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: {{ $restartPolicy }} + {{ tuple $envAll "ks_service" | include "helm-toolkit.snippets.kubernetes_image_pull_secrets" | indent 6 }} + nodeSelector: +{{ toYaml $nodeSelector | indent 8 }} +{{- if $tolerationsEnabled }} +{{ tuple $envAll $serviceName | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }} +{{- end}} + initContainers: +{{ tuple $envAll "ks_service" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: +{{- range $key1, $osServiceType := $serviceTypes }} + - name: {{ printf "%s-%s" $osServiceType "ks-service-registration" | quote }} + image: {{ $envAll.Values.images.tags.ks_service }} + imagePullPolicy: {{ $envAll.Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.ks_service | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + command: + - /bin/bash + - -c + - /tmp/ks-service.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: ks-service-sh + mountPath: /tmp/ks-service.sh + subPath: ks-service.sh + readOnly: true +{{ dict "enabled" true "name" $tlsSecret "ca" true | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }} + env: +{{- with $env := dict "ksUserSecret" $envAll.Values.secrets.identity.admin "useCA" (ne $tlsSecret "") }} +{{- include "helm-toolkit.snippets.keystone_openrc_env_vars" $env | indent 12 }} +{{- end }} + - name: OS_SERVICE_NAME + value: {{ tuple $osServiceType $envAll | include "helm-toolkit.endpoints.keystone_endpoint_name_lookup" }} + - name: OS_SERVICE_TYPE + value: {{ $osServiceType | quote }} +{{- end }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: ks-service-sh +{{- if $secretBin }} + secret: + secretName: {{ $secretBin | quote }} + defaultMode: 0555 +{{- else }} + configMap: + name: {{ $configMapBin | quote }} + defaultMode: 0555 +{{- end }} +{{- dict "enabled" true "name" $tlsSecret | include "helm-toolkit.snippets.tls_volume" | indent 8 }} +{{- end }} diff --git a/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl b/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl new file mode 100644 index 0000000000..58dcdc5c6d --- /dev/null +++ b/helm-toolkit/templates/manifests/_job-ks-user.yaml.tpl @@ -0,0 +1,155 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# This function creates a manifest for keystone user management. +# It can be used in charts dict created similar to the following: +# {- $ksUserJob := dict "envAll" . "serviceName" "senlin" } +# { $ksUserJob | include "helm-toolkit.manifests.job_ks_user" } + +{{/* + # To enable PodSecuritycontext (PodSecurityContext/v1) define the below in values.yaml: + # example: + # values: | + # pod: + # security_context: + # ks_user: + # pod: + # runAsUser: 65534 + # To enable Container SecurityContext(SecurityContext/v1) for ks-user container define the values: + # example: + # values: | + # pod: + # security_context: + # ks_user: + # container: + # ks-user: + # runAsUser: 65534 + # readOnlyRootFilesystem: true + # allowPrivilegeEscalation: false +*/}} + +{{- define "helm-toolkit.manifests.job_ks_user" -}} +{{- $envAll := index . "envAll" -}} +{{- $serviceName := index . "serviceName" -}} +{{- $jobAnnotations := index . "jobAnnotations" -}} +{{- $jobLabels := index . "jobLabels" -}} +{{- $nodeSelector := index . "nodeSelector" | default ( dict $envAll.Values.labels.job.node_selector_key $envAll.Values.labels.job.node_selector_value ) -}} +{{- $tolerationsEnabled := index . "tolerationsEnabled" | default false -}} +{{- $configMapBin := index . "configMapBin" | default (printf "%s-%s" $serviceName "bin" ) -}} +{{- $serviceUser := index . "serviceUser" | default $serviceName -}} +{{- $secretBin := index . "secretBin" -}} +{{- $tlsSecret := index . "tlsSecret" | default "" -}} +{{- $backoffLimit := index . "backoffLimit" | default "1000" -}} +{{- $activeDeadlineSeconds := index . "activeDeadlineSeconds" -}} +{{- $serviceUserPretty := $serviceUser | replace "_" "-" -}} +{{- $restartPolicy_ := "OnFailure" -}} +{{- if hasKey $envAll.Values "jobs" -}} +{{- if hasKey $envAll.Values.jobs "ks_user" -}} +{{- $restartPolicy_ = $envAll.Values.jobs.ks_user.restartPolicy | default $restartPolicy_ }} +{{- end }} +{{- end }} +{{- $restartPolicy := index . "restartPolicy" | default $restartPolicy_ -}} + +{{- $serviceAccountName := printf "%s-%s" $serviceUserPretty "ks-user" }} +{{ tuple $envAll "ks_user" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ printf "%s-%s" $serviceUserPretty "ks-user" | quote }} + labels: +{{ tuple $envAll $serviceName "ks-user" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- if $jobLabels }} +{{ toYaml $jobLabels | indent 4 }} +{{- end }} + annotations: +{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}} +{{- if $jobAnnotations }} +{{ toYaml $jobAnnotations | indent 4 }} +{{- end }} +spec: + backoffLimit: {{ $backoffLimit }} +{{- if $activeDeadlineSeconds }} + activeDeadlineSeconds: {{ $activeDeadlineSeconds }} +{{- end }} + template: + metadata: + labels: +{{ tuple $envAll $serviceName "ks-user" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} +{{- if $jobLabels }} +{{ toYaml $jobLabels | indent 8 }} +{{- end }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + spec: + serviceAccountName: {{ $serviceAccountName | quote }} +{{ dict "envAll" $envAll "application" "ks_user" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + restartPolicy: {{ $restartPolicy }} + {{ tuple $envAll "ks_user" | include "helm-toolkit.snippets.kubernetes_image_pull_secrets" | indent 6 }} + nodeSelector: +{{ toYaml $nodeSelector | indent 8 }} +{{- if $tolerationsEnabled }} +{{ tuple $envAll $serviceName | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }} +{{- end}} + initContainers: +{{ tuple $envAll "ks_user" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: ks-user + image: {{ $envAll.Values.images.tags.ks_user }} + imagePullPolicy: {{ $envAll.Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.ks_user | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "ks_user" "container" "ks_user" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /bin/bash + - -c + - /tmp/ks-user.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: ks-user-sh + mountPath: /tmp/ks-user.sh + subPath: ks-user.sh + readOnly: true +{{ dict "enabled" true "name" $tlsSecret "ca" true | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }} + env: +{{- with $env := dict "ksUserSecret" $envAll.Values.secrets.identity.admin "useCA" (ne $tlsSecret "") }} +{{- include "helm-toolkit.snippets.keystone_openrc_env_vars" $env | indent 12 }} +{{- end }} + - name: SERVICE_OS_SERVICE_NAME + value: {{ $serviceName | quote }} +{{- with $env := dict "ksUserSecret" (index $envAll.Values.secrets.identity $serviceUser ) }} +{{- include "helm-toolkit.snippets.keystone_user_create_env_vars" $env | indent 12 }} +{{- end }} + - name: SERVICE_OS_ROLES + {{- $serviceOsRoles := index $envAll.Values.endpoints.identity.auth $serviceUser "role" }} + {{- if kindIs "slice" $serviceOsRoles }} + value: {{ include "helm-toolkit.utils.joinListWithComma" $serviceOsRoles | quote }} + {{- else }} + value: {{ $serviceOsRoles | quote }} + {{- end }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: ks-user-sh +{{- if $secretBin }} + secret: + secretName: {{ $secretBin | quote }} + defaultMode: 0555 +{{- else }} + configMap: + name: {{ $configMapBin | quote }} + defaultMode: 0555 +{{- end }} +{{- dict "enabled" true "name" $tlsSecret | include "helm-toolkit.snippets.tls_volume" | indent 8 }} +{{- end -}} diff --git a/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl b/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl new file mode 100644 index 0000000000..2cfadafe32 --- /dev/null +++ b/helm-toolkit/templates/manifests/_job-rabbit-init.yaml.tpl @@ -0,0 +1,130 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "helm-toolkit.manifests.job_rabbit_init" -}} +{{- $envAll := index . "envAll" -}} +{{- $serviceName := index . "serviceName" -}} +{{- $jobAnnotations := index . "jobAnnotations" -}} +{{- $jobLabels := index . "jobLabels" -}} +{{- $nodeSelector := index . "nodeSelector" | default ( dict $envAll.Values.labels.job.node_selector_key $envAll.Values.labels.job.node_selector_value ) -}} +{{- $tolerationsEnabled := index . "tolerationsEnabled" | default false -}} +{{- $configMapBin := index . "configMapBin" | default (printf "%s-%s" $serviceName "bin" ) -}} +{{- $serviceUser := index . "serviceUser" | default $serviceName -}} +{{- $secretBin := index . "secretBin" -}} +{{- $backoffLimit := index . "backoffLimit" | default "1000" -}} +{{- $activeDeadlineSeconds := index . "activeDeadlineSeconds" -}} +{{- $serviceUserPretty := $serviceUser | replace "_" "-" -}} +{{- $serviceNamePretty := $serviceName | replace "_" "-" -}} +{{- $tlsPath := index . "tlsPath" | default "/etc/rabbitmq/certs" -}} +{{- $tlsSecret := index . "tlsSecret" | default "" -}} + +{{- $serviceAccountName := printf "%s-%s" $serviceUserPretty "rabbit-init" }} +{{ tuple $envAll "rabbit_init" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ printf "%s-%s" $serviceUserPretty "rabbit-init" | quote }} + labels: +{{ tuple $envAll $serviceName "rabbit-init" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- if $jobLabels }} +{{ toYaml $jobLabels | indent 4 }} +{{- end }} + annotations: +{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}} +{{- if $jobAnnotations }} +{{ toYaml $jobAnnotations | indent 4 }} +{{- end }} +spec: + backoffLimit: {{ $backoffLimit }} +{{- if $activeDeadlineSeconds }} + activeDeadlineSeconds: {{ $activeDeadlineSeconds }} +{{- end }} + template: + metadata: + labels: +{{ tuple $envAll $serviceName "rabbit-init" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} +{{- if $jobLabels }} +{{ toYaml $jobLabels | indent 8 }} +{{- end }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + spec: + serviceAccountName: {{ $serviceAccountName | quote }} + restartPolicy: OnFailure + {{ tuple $envAll "rabbit_init" | include "helm-toolkit.snippets.kubernetes_image_pull_secrets" | indent 6 }} + nodeSelector: +{{ toYaml $nodeSelector | indent 8 }} +{{- if $tolerationsEnabled }} +{{ tuple $envAll $serviceName | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }} +{{- end}} + initContainers: +{{ tuple $envAll "rabbit_init" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: rabbit-init + image: {{ $envAll.Values.images.tags.rabbit_init | quote }} + imagePullPolicy: {{ $envAll.Values.images.pull_policy | quote }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.rabbit_init | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + command: + - /bin/bash + - -c + - /tmp/rabbit-init.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: rabbit-init-sh + mountPath: /tmp/rabbit-init.sh + subPath: rabbit-init.sh + readOnly: true +{{- if $envAll.Values.manifests.certificates }} +{{- dict "enabled" $envAll.Values.manifests.certificates "name" $tlsSecret "path" $tlsPath | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }} +{{- end }} + env: + - name: RABBITMQ_ADMIN_CONNECTION + valueFrom: + secretKeyRef: + name: {{ $envAll.Values.secrets.oslo_messaging.admin }} + key: RABBITMQ_CONNECTION + - name: RABBITMQ_USER_CONNECTION + valueFrom: + secretKeyRef: + name: {{ index $envAll.Values.secrets.oslo_messaging $serviceName }} + key: RABBITMQ_CONNECTION +{{- if $envAll.Values.conf.rabbitmq }} + - name: RABBITMQ_AUXILIARY_CONFIGURATION + value: {{ toJson $envAll.Values.conf.rabbitmq | quote }} +{{- end }} +{{- if and $envAll.Values.manifests.certificates (ne $tlsSecret "") }} + - name: RABBITMQ_X509 + value: "REQUIRE X509" + - name: USER_CERT_PATH + value: {{ $tlsPath | quote }} +{{- end }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: rabbit-init-sh +{{- if $secretBin }} + secret: + secretName: {{ $secretBin | quote }} + defaultMode: 0555 +{{- else }} + configMap: + name: {{ $configMapBin | quote }} + defaultMode: 0555 +{{- end }} +{{- if $envAll.Values.manifests.certificates }} +{{- dict "enabled" $envAll.Values.manifests.certificates "name" $tlsSecret | include "helm-toolkit.snippets.tls_volume" | indent 8 }} +{{- end }} +{{- end -}} diff --git a/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl b/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl new file mode 100644 index 0000000000..b5fdc09c32 --- /dev/null +++ b/helm-toolkit/templates/manifests/_job-s3-bucket.yaml.tpl @@ -0,0 +1,148 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# This function creates a manifest for linking an s3 bucket to an s3 user. +# It can be used in charts dict created similar to the following: +# {- $s3BucketJob := dict "envAll" . "serviceName" "elasticsearch" } +# { $s3BucketJob | include "helm-toolkit.manifests.job_s3_bucket" } + +{{- define "helm-toolkit.manifests.job_s3_bucket" -}} +{{- $envAll := index . "envAll" -}} +{{- $serviceName := index . "serviceName" -}} +{{- $jobAnnotations := index . "jobAnnotations" -}} +{{- $jobLabels := index . "jobLabels" -}} +{{- $nodeSelector := index . "nodeSelector" | default ( dict $envAll.Values.labels.job.node_selector_key $envAll.Values.labels.job.node_selector_value ) -}} +{{- $tolerationsEnabled := index . "tolerationsEnabled" | default false -}} +{{- $configMapBin := index . "configMapBin" | default (printf "%s-%s" $serviceName "bin" ) -}} +{{- $configMapCeph := index . "configMapCeph" | default (printf "ceph-etc" ) -}} +{{- $secretBin := index . "secretBin" -}} +{{- $backoffLimit := index . "backoffLimit" | default "1000" -}} +{{- $activeDeadlineSeconds := index . "activeDeadlineSeconds" -}} +{{- $serviceNamePretty := $serviceName | replace "_" "-" -}} +{{- $s3UserSecret := index $envAll.Values.secrets.rgw $serviceName -}} +{{- $s3Bucket := index . "s3Bucket" | default $serviceName }} +{{- $tlsCertificateSecret := index . "tlsCertificateSecret" -}} +{{- $tlsCertificatePath := index . "tlsCertificatePath" -}} + +{{- $serviceAccountName := printf "%s-%s" $serviceNamePretty "s3-bucket" }} +{{ tuple $envAll "s3_bucket" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ printf "%s-%s" $serviceNamePretty "s3-bucket" | quote }} + labels: +{{ tuple $envAll $serviceName "s3-bucket" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- if $jobLabels }} +{{ toYaml $jobLabels | indent 4 }} +{{- end }} + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}} +{{- if $jobAnnotations }} +{{ toYaml $jobAnnotations | indent 4 }} +{{- end }} +spec: + backoffLimit: {{ $backoffLimit }} +{{- if $activeDeadlineSeconds }} + activeDeadlineSeconds: {{ $activeDeadlineSeconds }} +{{- end }} + template: + metadata: + labels: +{{ tuple $envAll $serviceName "s3-bucket" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} +{{- if $jobLabels }} +{{ toYaml $jobLabels | indent 8 }} +{{- end }} + spec: + serviceAccountName: {{ $serviceAccountName | quote }} + restartPolicy: OnFailure + {{ tuple $envAll "s3_bucket" | include "helm-toolkit.snippets.kubernetes_image_pull_secrets" | indent 6 }} + nodeSelector: +{{ toYaml $nodeSelector | indent 8 }} +{{- if $tolerationsEnabled }} +{{ tuple $envAll $serviceName | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }} +{{- end}} + initContainers: +{{ tuple $envAll "s3_bucket" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: s3-bucket + image: {{ $envAll.Values.images.tags.s3_bucket }} + imagePullPolicy: {{ $envAll.Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.s3_bucket | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + command: + - /bin/bash + - -c + - /tmp/create-s3-bucket.sh + env: +{{- with $env := dict "s3AdminSecret" $envAll.Values.secrets.rgw.admin }} +{{- include "helm-toolkit.snippets.rgw_s3_admin_env_vars" $env | indent 12 }} +{{- end }} +{{- include "helm-toolkit.snippets.rgw_s3_user_env_vars" $envAll | indent 12 }} + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: s3-bucket-sh + mountPath: /tmp/create-s3-bucket.sh + subPath: create-s3-bucket.sh + readOnly: true + - name: etcceph + mountPath: /etc/ceph + - name: ceph-etc + mountPath: /etc/ceph/ceph.conf + subPath: ceph.conf + readOnly: true + {{- if empty $envAll.Values.conf.ceph.admin_keyring }} + - name: ceph-keyring + mountPath: /tmp/client-keyring + subPath: key + readOnly: true + {{ end }} +{{- if and ($tlsCertificatePath) ($tlsCertificateSecret) }} + - name: {{ $tlsCertificateSecret }} + mountPath: {{ $tlsCertificatePath }} + subPath: ca.crt + readOnly: true +{{- end }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: s3-bucket-sh +{{- if $secretBin }} + secret: + secretName: {{ $secretBin | quote }} + defaultMode: 0555 +{{- else }} + configMap: + name: {{ $configMapBin | quote }} + defaultMode: 0555 +{{- end }} + - name: etcceph + emptyDir: {} + - name: ceph-etc + configMap: + name: {{ $configMapCeph | quote }} + defaultMode: 0444 + {{- if empty $envAll.Values.conf.ceph.admin_keyring }} + - name: ceph-keyring + secret: + secretName: pvc-ceph-client-key + {{ end }} +{{- if and ($tlsCertificatePath) ($tlsCertificateSecret) }} + - name: {{ $tlsCertificateSecret }} + secret: + secretName: {{ $tlsCertificateSecret }} + defaultMode: 292 +{{- end }} +{{- end -}} diff --git a/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl b/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl new file mode 100644 index 0000000000..77d1a71e98 --- /dev/null +++ b/helm-toolkit/templates/manifests/_job-s3-user.yaml.tpl @@ -0,0 +1,160 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# This function creates a manifest for s3 user management. +# It can be used in charts dict created similar to the following: +# {- $s3UserJob := dict "envAll" . "serviceName" "elasticsearch" } +# { $s3UserJob | include "helm-toolkit.manifests.job_s3_user" } + +{{- define "helm-toolkit.manifests.job_s3_user" -}} +{{- $envAll := index . "envAll" -}} +{{- $serviceName := index . "serviceName" -}} +{{- $jobAnnotations := index . "jobAnnotations" -}} +{{- $jobLabels := index . "jobLabels" -}} +{{- $nodeSelector := index . "nodeSelector" | default ( dict $envAll.Values.labels.job.node_selector_key $envAll.Values.labels.job.node_selector_value ) -}} +{{- $tolerationsEnabled := index . "tolerationsEnabled" | default false -}} +{{- $configMapBin := index . "configMapBin" | default (printf "%s-%s" $serviceName "bin" ) -}} +{{- $configMapCeph := index . "configMapCeph" | default (printf "ceph-etc" ) -}} +{{- $secretBin := index . "secretBin" -}} +{{- $backoffLimit := index . "backoffLimit" | default "1000" -}} +{{- $activeDeadlineSeconds := index . "activeDeadlineSeconds" -}} +{{- $serviceNamePretty := $serviceName | replace "_" "-" -}} +{{- $s3UserSecret := index $envAll.Values.secrets.rgw $serviceName -}} + +{{- $serviceAccountName := printf "%s-%s" $serviceNamePretty "s3-user" }} +{{ tuple $envAll "s3_user" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ printf "%s-%s" $serviceNamePretty "s3-user" | quote }} + labels: +{{ tuple $envAll $serviceName "s3-user" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- if $jobLabels }} +{{ toYaml $jobLabels | indent 4 }} +{{- end }} + annotations: + "helm.sh/hook-delete-policy": before-hook-creation + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +{{ tuple $serviceAccountName $envAll | include "helm-toolkit.snippets.custom_job_annotations" | indent 4 -}} +{{- if $jobAnnotations }} +{{ toYaml $jobAnnotations | indent 4 }} +{{- end }} +spec: + backoffLimit: {{ $backoffLimit }} +{{- if $activeDeadlineSeconds }} + activeDeadlineSeconds: {{ $activeDeadlineSeconds }} +{{- end }} + template: + metadata: + labels: +{{ tuple $envAll $serviceName "s3-user" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} +{{- if $jobLabels }} +{{ toYaml $jobLabels | indent 8 }} +{{- end }} + spec: + serviceAccountName: {{ $serviceAccountName | quote }} + restartPolicy: OnFailure + {{ tuple $envAll "s3_user" | include "helm-toolkit.snippets.kubernetes_image_pull_secrets" | indent 6 }} + nodeSelector: +{{ toYaml $nodeSelector | indent 8 }} +{{- if $tolerationsEnabled }} +{{ tuple $envAll $serviceName | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }} +{{- end}} + initContainers: +{{ tuple $envAll "s3_user" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + - name: ceph-keyring-placement + image: {{ $envAll.Values.images.tags.ceph_key_placement }} + imagePullPolicy: {{ $envAll.Values.images.pull_policy }} + command: + - /tmp/ceph-admin-keyring.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: etcceph + mountPath: /etc/ceph + - name: ceph-keyring-sh + mountPath: /tmp/ceph-admin-keyring.sh + subPath: ceph-admin-keyring.sh + readOnly: true + {{- if empty $envAll.Values.conf.ceph.admin_keyring }} + - name: ceph-keyring + mountPath: /tmp/client-keyring + subPath: key + readOnly: true + {{ end }} + containers: + - name: s3-user + image: {{ $envAll.Values.images.tags.s3_user }} + imagePullPolicy: {{ $envAll.Values.images.pull_policy }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.s3_user | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + command: + - /bin/bash + - -c + - /tmp/create-s3-user.sh + env: +{{- with $env := dict "s3AdminSecret" $envAll.Values.secrets.rgw.admin }} +{{- include "helm-toolkit.snippets.rgw_s3_admin_env_vars" $env | indent 12 }} +{{- end }} +{{- include "helm-toolkit.snippets.rgw_s3_user_env_vars" $envAll | indent 12 }} + - name: RGW_HOST + value: {{ tuple "ceph_object_store" "internal" "api" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: create-s3-user-sh + mountPath: /tmp/create-s3-user.sh + subPath: create-s3-user.sh + readOnly: true + - name: etcceph + mountPath: /etc/ceph + - name: ceph-etc + mountPath: /etc/ceph/ceph.conf + subPath: ceph.conf + readOnly: true + {{- if empty $envAll.Values.conf.ceph.admin_keyring }} + - name: ceph-keyring + mountPath: /tmp/client-keyring + subPath: key + readOnly: true + {{ end }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: create-s3-user-sh +{{- if $secretBin }} + secret: + secretName: {{ $secretBin | quote }} + defaultMode: 0555 +{{- else }} + configMap: + name: {{ $configMapBin | quote }} + defaultMode: 0555 +{{- end }} + - name: ceph-keyring-sh + configMap: + name: {{ $configMapBin | quote }} + defaultMode: 0555 + - name: etcceph + emptyDir: {} + - name: ceph-etc + configMap: + name: {{ $configMapCeph | quote }} + defaultMode: 0444 + {{- if empty $envAll.Values.conf.ceph.admin_keyring }} + - name: ceph-keyring + secret: + secretName: pvc-ceph-client-key + {{ end }} +{{- end -}} diff --git a/helm-toolkit/templates/manifests/_job_image_repo_sync.tpl b/helm-toolkit/templates/manifests/_job_image_repo_sync.tpl new file mode 100644 index 0000000000..0906df4c9e --- /dev/null +++ b/helm-toolkit/templates/manifests/_job_image_repo_sync.tpl @@ -0,0 +1,119 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# This function creates a manifest for the image repo sync jobs. +# It can be used in charts dict created similar to the following: +# {- $imageRepoSyncJob := dict "envAll" . "serviceName" "prometheus" -} +# { $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" } + +{{- define "helm-toolkit.manifests.job_image_repo_sync" -}} +{{- $envAll := index . "envAll" -}} +{{- $serviceName := index . "serviceName" -}} +{{- $jobAnnotations := index . "jobAnnotations" -}} +{{- $jobLabels := index . "jobLabels" -}} +{{- $nodeSelector := index . "nodeSelector" | default ( dict $envAll.Values.labels.job.node_selector_key $envAll.Values.labels.job.node_selector_value ) -}} +{{- $tolerationsEnabled := index . "tolerationsEnabled" | default false -}} +{{- $podVolMounts := index . "podVolMounts" | default false -}} +{{- $podVols := index . "podVols" | default false -}} +{{- $configMapBin := index . "configMapBin" | default (printf "%s-%s" $serviceName "bin" ) -}} +{{- $secretBin := index . "secretBin" -}} +{{- $backoffLimit := index . "backoffLimit" | default "1000" -}} +{{- $activeDeadlineSeconds := index . "activeDeadlineSeconds" -}} +{{- $serviceNamePretty := $serviceName | replace "_" "-" -}} + +{{- $serviceAccountName := printf "%s-%s" $serviceNamePretty "image-repo-sync" }} +{{ tuple $envAll "image_repo_sync" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ printf "%s-%s" $serviceNamePretty "image-repo-sync" | quote }} + labels: +{{ tuple $envAll $serviceName "image-repo-sync" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- if $jobLabels }} +{{ toYaml $jobLabels | indent 4 }} +{{- end }} + annotations: + "helm.sh/hook-delete-policy": before-hook-creation +{{- if $jobAnnotations }} +{{ toYaml $jobAnnotations | indent 4 }} +{{- end }} +spec: + backoffLimit: {{ $backoffLimit }} +{{- if $activeDeadlineSeconds }} + activeDeadlineSeconds: {{ $activeDeadlineSeconds }} +{{- end }} + template: + metadata: + labels: +{{ tuple $envAll $serviceName "image-repo-sync" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} +{{- if $jobLabels }} +{{ toYaml $jobLabels | indent 8 }} +{{- end }} + spec: + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + {{ tuple $envAll "image_repo_sync" | include "helm-toolkit.snippets.kubernetes_image_pull_secrets" | indent 6 }} + nodeSelector: +{{ toYaml $nodeSelector | indent 8 }} +{{- if $tolerationsEnabled }} +{{ tuple $envAll $serviceName | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }} +{{- end}} + initContainers: +{{ tuple $envAll "image_repo_sync" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: image-repo-sync +{{ tuple $envAll "image_repo_sync" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.image_repo_sync | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + env: + - name: LOCAL_REPO + value: "{{ tuple "local_image_registry" "node" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}:{{ tuple "local_image_registry" "node" "registry" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }}" + - name: IMAGE_SYNC_LIST + value: "{{ include "helm-toolkit.utils.image_sync_list" $envAll }}" + command: + - /bin/bash + - -c + - /tmp/image-repo-sync.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: bootstrap-sh + mountPath: /tmp/image-repo-sync.sh + subPath: image-repo-sync.sh + readOnly: true + - name: docker-socket + mountPath: /var/run/docker.sock +{{- if $podVolMounts }} +{{ $podVolMounts | toYaml | indent 12 }} +{{- end }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: bootstrap-sh +{{- if $secretBin }} + secret: + secretName: {{ $secretBin | quote }} + defaultMode: 0555 +{{- else }} + configMap: + name: {{ $configMapBin | quote }} + defaultMode: 0555 +{{- end }} + - name: docker-socket + hostPath: + path: /var/run/docker.sock +{{- if $podVols }} +{{ $podVols | toYaml | indent 8 }} +{{- end }} +{{- end }} diff --git a/helm-toolkit/templates/manifests/_network_policy.tpl b/helm-toolkit/templates/manifests/_network_policy.tpl new file mode 100644 index 0000000000..ae074502b1 --- /dev/null +++ b/helm-toolkit/templates/manifests/_network_policy.tpl @@ -0,0 +1,281 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Creates a network policy manifest for services. +values: | + endpoints: + kube_dns: + namespace: kube-system + name: kubernetes-dns + hosts: + default: kube-dns + host_fqdn_override: + default: null + path: + default: null + scheme: http + port: + dns_tcp: + default: 53 + dns: + default: 53 + protocol: UDP + network_policy: + myLabel: + podSelector: + matchLabels: + component: api + ingress: + - from: + - podSelector: + matchLabels: + application: keystone + ports: + - protocol: TCP + port: 80 + egress: + - to: + - namespaceSelector: + matchLabels: + name: default + - namespaceSelector: + matchLabels: + name: kube-public + ports: + - protocol: TCP + port: 53 + - protocol: UDP + port: 53 +usage: | + {{ dict "envAll" . "name" "application" "label" "myLabel" | include "helm-toolkit.manifests.kubernetes_network_policy" }} + {{ dict "envAll" . "key" "myLabel" "labels" (dict "application" "myApp" "component" "myComp")}} +return: | + --- + apiVersion: networking.k8s.io/v1 + kind: NetworkPolicy + metadata: + name: RELEASE-NAME + namespace: NAMESPACE + spec: + policyTypes: + - Ingress + - Egress + podSelector: + matchLabels: + application: myLabel + component: api + ingress: + - from: + - podSelector: + matchLabels: + application: keystone + ports: + - protocol: TCP + port: 80 + egress: + - to: + - podSelector: + matchLabels: + name: default + - namespaceSelector: + matchLabels: + name: kube-public + ports: + - protocol: TCP + port: 53 + - protocol: UDP + port: 53 + --- + apiVersion: networking.k8s.io/v1 + kind: NetworkPolicy + metadata: + name: RELEASE-NAME + namespace: NAMESPACE + spec: + policyTypes: + - Ingress + - Egress + podSelector: + matchLabels: + application: myApp + component: myComp + ingress: + - from: + - podSelector: + matchLabels: + application: keystone + ports: + - protocol: TCP + port: 80 + egress: + - to: + - podSelector: + matchLabels: + name: default + - namespaceSelector: + matchLabels: + name: kube-public + ports: + - protocol: TCP + port: 53 + - protocol: UDP + port: 53 +*/}} + +{{/* +abstract: | + Creates a network policy manifest for services. +values: | + network_policy: + myLabel: + spec: + +usage: | + {{ dict "envAll" . "name" "application" "label" "myLabel" | include "helm-toolkit.manifests.kubernetes_network_policy" }} + +return: | + --- + apiVersion: networking.k8s.io/v1 + kind: NetworkPolicy + metadata: + name: RELEASE-NAME-myLabel-netpol + namespace: NAMESPACE + spec: + +*/}} + +{{- define "helm-toolkit.manifests.kubernetes_network_policy" -}} +{{- $envAll := index . "envAll" -}} +{{- $name := index . "name" -}} +{{- $labels := index . "labels" | default nil -}} +{{- $label := index . "key" | default (index . "label") -}} + +{{- $spec_labels := list -}} +{{- range $label, $value := $envAll.Values.network_policy }} +{{- if hasKey $value "spec" }} +{{- $spec_labels = append $spec_labels $label }} +{{- end }} +{{- end }} +{{- if $spec_labels }} +{{- range $label := $spec_labels }} +{{- $raw_spec := (index $envAll.Values.network_policy $label "spec") }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ $envAll.Release.Name }}-{{ $label | replace "_" "-" }}-netpol + namespace: {{ $envAll.Release.Namespace }} +spec: +{{ $raw_spec | toYaml | indent 2 }} +{{- end }} +{{- else }} +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: {{ $label | replace "_" "-" }}-netpol + namespace: {{ $envAll.Release.Namespace }} +spec: +{{- if hasKey (index $envAll.Values "network_policy") $label }} + policyTypes: +{{- $is_egress := false -}} +{{- if hasKey (index $envAll.Values.network_policy $label) "policyTypes" -}} +{{- if has "Egress" (index $envAll.Values.network_policy $label "policyTypes") -}} +{{- $is_egress = true -}} +{{- end -}} +{{- end -}} +{{- if or $is_egress (index $envAll.Values.network_policy $label "egress") }} + - Egress +{{ end -}} +{{- $is_ingress := false -}} +{{- if hasKey (index $envAll.Values.network_policy $label) "policyTypes" -}} +{{- if has "Ingress" (index $envAll.Values.network_policy $label "policyTypes") -}} +{{- $is_ingress = true -}} +{{- end -}} +{{- end -}} +{{- if or $is_ingress (index $envAll.Values.network_policy $label "ingress") }} + - Ingress +{{ end -}} +{{- end }} + podSelector: + matchLabels: +{{- if empty $labels }} + {{ $name }}: {{ $label }} +{{- else }} +{{ range $k, $v := $labels }} + {{ $k }}: {{ $v }} +{{- end }} +{{- end }} +{{- if hasKey (index $envAll.Values "network_policy") $label }} +{{- if hasKey (index $envAll.Values.network_policy $label) "podSelector" }} +{{- if index $envAll.Values.network_policy $label "podSelector" "matchLabels" }} +{{ index $envAll.Values.network_policy $label "podSelector" "matchLabels" | toYaml | indent 6 }} +{{ end }} +{{ end }} +{{ end }} +{{- if hasKey (index $envAll.Values "network_policy") $label }} + egress: +{{- range $key, $value := $envAll.Values.endpoints }} +{{- if kindIs "map" $value }} +{{- if or (hasKey $value "namespace") (hasKey $value "hosts") }} + - to: +{{- if index $value "namespace" }} + - namespaceSelector: + matchLabels: + name: {{ index $value "namespace" }} +{{- else if index $value "hosts" }} +{{- $defaultValue := index $value "hosts" "internal" }} +{{- if hasKey (index $value "hosts") "internal" }} +{{- $a := split "-" $defaultValue }} + - podSelector: + matchLabels: + application: {{ printf "%s" (index $a._0) | default $defaultValue }} +{{- else }} +{{- $defaultValue := index $value "hosts" "default" }} +{{- $a := split "-" $defaultValue }} + - podSelector: + matchLabels: + application: {{ printf "%s" (index $a._0) | default $defaultValue }} +{{- end }} +{{- end }} +{{- if index $value "port" }} + ports: +{{- range $k, $v := index $value "port" }} +{{- if $k }} +{{- range $pk, $pv := $v }} +{{- if and $pv (ne $pk "protocol") }} + - port: {{ $pv }} + protocol: {{ $v.protocol | default "TCP" }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- if index $envAll.Values.network_policy $label "egress" }} +{{ index $envAll.Values.network_policy $label "egress" | toYaml | indent 4 }} +{{- end }} +{{- end }} +{{- if hasKey (index $envAll.Values "network_policy") $label }} +{{- if index $envAll.Values.network_policy $label "ingress" }} + ingress: +{{ index $envAll.Values.network_policy $label "ingress" | toYaml | indent 4 }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl b/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl new file mode 100644 index 0000000000..7ad505b558 --- /dev/null +++ b/helm-toolkit/templates/manifests/_secret-registry.yaml.tpl @@ -0,0 +1,78 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Creates a manifest for a authenticating a registry with a secret +examples: + - values: | + annotations: + secret: + oci_image_registry: + {{ $serviceName }}: + custom.tld/key: "value" + secrets: + oci_image_registry: + {{ $serviceName }}: {{ $keyName }} + endpoints: + oci_image_registry: + name: oci-image-registry + auth: + enabled: true + {{ $serviceName }}: + name: {{ $userName }} + password: {{ $password }} + usage: | + {{- include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) -}} + return: | + --- + apiVersion: v1 + kind: Secret + metadata: + name: {{ $secretName }} + annotations: + custom.tld/key: "value" + type: kubernetes.io/dockerconfigjson + data: + dockerconfigjson: {{ $dockerAuth }} +*/}} + +{{- define "helm-toolkit.manifests.secret_registry" }} +{{- $envAll := index . "envAll" }} +{{- $registryUser := index . "registryUser" }} +{{- $secretName := index $envAll.Values.secrets.oci_image_registry $registryUser }} +{{- $registryHost := tuple "oci_image_registry" "internal" $envAll | include "helm-toolkit.endpoints.endpoint_host_lookup" }} +{{/* +We only use "host:port" when port is non-null, else just use "host" +*/}} +{{- $registryPort := "" }} +{{- $port := $envAll.Values.endpoints.oci_image_registry.port.registry.default }} +{{- if $port }} +{{- $port = tuple "oci_image_registry" "internal" "registry" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +{{- $registryPort = printf ":%s" $port }} +{{- end }} +{{- $imageCredentials := index $envAll.Values.endpoints.oci_image_registry.auth $registryUser }} +{{- $dockerAuthToken := printf "%s:%s" $imageCredentials.username $imageCredentials.password | b64enc }} +{{- $dockerAuth := printf "{\"auths\": {\"%s%s\": {\"auth\": \"%s\"}}}" $registryHost $registryPort $dockerAuthToken | b64enc }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} + annotations: +{{ tuple "oci_image_registry" $registryUser $envAll | include "helm-toolkit.snippets.custom_secret_annotations" | indent 4 }} +type: kubernetes.io/dockerconfigjson +data: + .dockerconfigjson: {{ $dockerAuth }} +{{- end -}} diff --git a/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl b/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl new file mode 100644 index 0000000000..c800340306 --- /dev/null +++ b/helm-toolkit/templates/manifests/_secret-tls.yaml.tpl @@ -0,0 +1,119 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Creates a manifest for a services public tls secret +examples: + - values: | + annotations: + secret: + tls: + key_manager_api_public: + custom.tld/key: "value" + secrets: + tls: + key_manager: + api: + public: barbican-tls-public + endpoints: + key_manager: + host_fqdn_override: + public: + tls: + crt: | + FOO-CRT + key: | + FOO-KEY + ca: | + FOO-CA_CRT + usage: | + {{- include "helm-toolkit.manifests.secret_ingress_tls" ( dict "envAll" . "backendServiceType" "key-manager" ) -}} + return: | + --- + apiVersion: v1 + kind: Secret + metadata: + name: barbican-tls-public + annotations: + custom.tld/key: "value" + type: kubernetes.io/tls + data: + tls.key: Rk9PLUtFWQo= + tls.crt: Rk9PLUNSVAoKRk9PLUNBX0NSVAo= + + - values: | + secrets: + tls: + key_manager: + api: + public: barbican-tls-public + endpoints: + key_manager: + host_fqdn_override: + public: + tls: + crt: | + FOO-CRT + FOO-INTERMEDIATE_CRT + FOO-CA_CRT + key: | + FOO-KEY + usage: | + {{- include "helm-toolkit.manifests.secret_ingress_tls" ( dict "envAll" . "backendServiceType" "key-manager" ) -}} + return: | + --- + apiVersion: v1 + kind: Secret + metadata: + name: barbican-tls-public + type: kubernetes.io/tls + data: + tls.key: Rk9PLUtFWQo= + tls.crt: Rk9PLUNSVApGT08tSU5URVJNRURJQVRFX0NSVApGT08tQ0FfQ1JUCg== +*/}} + +{{- define "helm-toolkit.manifests.secret_ingress_tls" }} +{{- $envAll := index . "envAll" }} +{{- $endpoint := index . "endpoint" | default "public" }} +{{- $backendServiceType := index . "backendServiceType" }} +{{- $backendService := index . "backendService" | default "api" }} +{{- $host := index $envAll.Values.endpoints ( $backendServiceType | replace "-" "_" ) "host_fqdn_override" }} +{{- if hasKey $host $endpoint }} +{{- $endpointHost := index $host $endpoint }} +{{- if kindIs "map" $endpointHost }} +{{- if hasKey $endpointHost "tls" }} +{{- if and $endpointHost.tls.key $endpointHost.tls.crt }} + +{{- $customAnnotationKey := printf "%s_%s_%s" ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ index $envAll.Values.secrets.tls ( $backendServiceType | replace "-" "_" ) $backendService $endpoint }} + annotations: +{{ tuple "tls" $customAnnotationKey $envAll | include "helm-toolkit.snippets.custom_secret_annotations" | indent 4 }} +type: kubernetes.io/tls +data: + tls.key: {{ $endpointHost.tls.key | b64enc }} +{{- if $endpointHost.tls.ca }} + tls.crt: {{ list $endpointHost.tls.crt $endpointHost.tls.ca | join "\n" | b64enc }} +{{- else }} + tls.crt: {{ $endpointHost.tls.crt | b64enc }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/helm-toolkit/templates/manifests/_service-ingress.tpl b/helm-toolkit/templates/manifests/_service-ingress.tpl new file mode 100644 index 0000000000..d2e7c0e8b0 --- /dev/null +++ b/helm-toolkit/templates/manifests/_service-ingress.tpl @@ -0,0 +1,43 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# This function creates a manifest for a services ingress rules. +# It can be used in charts dict created similar to the following: +# {- $serviceIngressOpts := dict "envAll" . "backendServiceType" "key-manager" -} +# { $serviceIngressOpts | include "helm-toolkit.manifests.service_ingress" } + +{{- define "helm-toolkit.manifests.service_ingress" -}} +{{- $envAll := index . "envAll" -}} +{{- $backendServiceType := index . "backendServiceType" -}} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple $backendServiceType "public" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +spec: + ports: + - name: http + port: 80 + - name: https + port: 443 + selector: + app: ingress-api +{{- if index $envAll.Values.endpoints $backendServiceType }} +{{- if index $envAll.Values.endpoints $backendServiceType "ip" }} +{{- if index $envAll.Values.endpoints $backendServiceType "ip" "ingress" }} + clusterIP: {{ (index $envAll.Values.endpoints $backendServiceType "ip" "ingress") }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/helm-toolkit/templates/scripts/_create-s3-bucket.sh.tpl b/helm-toolkit/templates/scripts/_create-s3-bucket.sh.tpl new file mode 100644 index 0000000000..bf1465b238 --- /dev/null +++ b/helm-toolkit/templates/scripts/_create-s3-bucket.sh.tpl @@ -0,0 +1,35 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{- define "helm-toolkit.scripts.create_s3_bucket" }} +#!/bin/bash +set -e +CONNECTION_ARGS="--host=$RGW_HOST --host-bucket=$RGW_HOST" +if [ "$RGW_PROTO" = "http" ]; then + CONNECTION_ARGS+=" --no-ssl" +else + CONNECTION_ARGS+=" --no-check-certificate" +fi +ADMIN_AUTH_ARGS=" --access_key=$S3_ADMIN_ACCESS_KEY --secret_key=$S3_ADMIN_SECRET_KEY" +USER_AUTH_ARGS=" --access_key=$S3_ACCESS_KEY --secret_key=$S3_SECRET_KEY" +function check_rgw_s3_bucket () { + s3cmd $CONNECTION_ARGS $USER_AUTH_ARGS ls s3://$S3_BUCKET +} +function create_rgw_s3_bucket () { + s3cmd $CONNECTION_ARGS $ADMIN_AUTH_ARGS mb s3://$S3_BUCKET +} +function modify_bucket_acl () { + s3cmd $CONNECTION_ARGS $ADMIN_AUTH_ARGS setacl s3://$S3_BUCKET --acl-grant=read:$S3_USERNAME --acl-grant=write:$S3_USERNAME +} +check_rgw_s3_bucket || ( create_rgw_s3_bucket && modify_bucket_acl ) +{{- end }} \ No newline at end of file diff --git a/helm-toolkit/templates/scripts/_create-s3-user.sh.tpl b/helm-toolkit/templates/scripts/_create-s3-user.sh.tpl new file mode 100644 index 0000000000..08796d29c0 --- /dev/null +++ b/helm-toolkit/templates/scripts/_create-s3-user.sh.tpl @@ -0,0 +1,65 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{- define "helm-toolkit.scripts.create_s3_user" }} +#!/bin/bash +set -e +function create_s3_user () { + echo "Creating s3 user and key pair" + radosgw-admin user create \ + --uid=${S3_USERNAME} \ + --display-name=${S3_USERNAME} \ + --key-type=s3 \ + --access-key ${S3_ACCESS_KEY} \ + --secret-key ${S3_SECRET_KEY} +} +function update_s3_user () { + # Retrieve old access keys, if they exist + old_access_keys=$(radosgw-admin user info --uid=${S3_USERNAME} \ + | jq -r '.keys[].access_key' || true) + + if [[ ! -z ${old_access_keys} ]]; then + for access_key in $old_access_keys; do + # If current access key is the same as the key supplied, do nothing. + if [ "$access_key" == "${S3_ACCESS_KEY}" ]; then + echo "Current user and key pair exists." + continue + else + # If keys differ, remove previous key + radosgw-admin key rm --uid=${S3_USERNAME} --key-type=s3 --access-key=$access_key + fi + done + fi + + # Perform one more additional check to account for scenarios where multiple + # key pairs existed previously, but one existing key was the supplied key + current_access_key=$(radosgw-admin user info --uid=${S3_USERNAME} \ + | jq -r '.keys[].access_key' || true) + + # If the supplied key does not exist, modify the user + if [[ -z ${current_access_key} ]]; then + # Modify user with new access and secret keys + echo "Updating existing user's key pair" + radosgw-admin user modify \ + --uid=${S3_USERNAME}\ + --access-key ${S3_ACCESS_KEY} \ + --secret-key ${S3_SECRET_KEY} + fi +} +user_exists=$(radosgw-admin user info --uid=${S3_USERNAME} || true) +if [[ -z ${user_exists} ]]; then + create_s3_user +else + update_s3_user +fi +{{- end }} \ No newline at end of file diff --git a/helm-toolkit/templates/scripts/_db-drop.py.tpl b/helm-toolkit/templates/scripts/_db-drop.py.tpl new file mode 100644 index 0000000000..c6a7521d29 --- /dev/null +++ b/helm-toolkit/templates/scripts/_db-drop.py.tpl @@ -0,0 +1,153 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "helm-toolkit.scripts.db_drop" }} +#!/usr/bin/env python + +# Drops db and user for an OpenStack Service: +# Set ROOT_DB_CONNECTION and DB_CONNECTION environment variables to contain +# SQLAlchemy strings for the root connection to the database and the one you +# wish the service to use. Alternatively, you can use an ini formatted config +# at the location specified by OPENSTACK_CONFIG_FILE, and extract the string +# from the key OPENSTACK_CONFIG_DB_KEY, in the section specified by +# OPENSTACK_CONFIG_DB_SECTION. + +import os +import sys +try: + import ConfigParser + PARSER_OPTS = {} +except ImportError: + import configparser as ConfigParser + PARSER_OPTS = {"strict": False} +import logging +from sqlalchemy import create_engine +from sqlalchemy import text + +# Create logger, console handler and formatter +logger = logging.getLogger('OpenStack-Helm DB Drop') +logger.setLevel(logging.DEBUG) +ch = logging.StreamHandler() +ch.setLevel(logging.DEBUG) +formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + +# Set the formatter and add the handler +ch.setFormatter(formatter) +logger.addHandler(ch) + + +# Get the connection string for the service db root user +if "ROOT_DB_CONNECTION" in os.environ: + db_connection = os.environ['ROOT_DB_CONNECTION'] + logger.info('Got DB root connection') +else: + logger.critical('environment variable ROOT_DB_CONNECTION not set') + sys.exit(1) + +mysql_x509 = os.getenv('MARIADB_X509', "") +ssl_args = {} +if mysql_x509: + ssl_args = {'ssl': {'ca': '/etc/mysql/certs/ca.crt', + 'key': '/etc/mysql/certs/tls.key', + 'cert': '/etc/mysql/certs/tls.crt'}} + +# Get the connection string for the service db +if "OPENSTACK_CONFIG_FILE" in os.environ: + os_conf = os.environ['OPENSTACK_CONFIG_FILE'] + if "OPENSTACK_CONFIG_DB_SECTION" in os.environ: + os_conf_section = os.environ['OPENSTACK_CONFIG_DB_SECTION'] + else: + logger.critical('environment variable OPENSTACK_CONFIG_DB_SECTION not set') + sys.exit(1) + if "OPENSTACK_CONFIG_DB_KEY" in os.environ: + os_conf_key = os.environ['OPENSTACK_CONFIG_DB_KEY'] + else: + logger.critical('environment variable OPENSTACK_CONFIG_DB_KEY not set') + sys.exit(1) + try: + config = ConfigParser.RawConfigParser(**PARSER_OPTS) + logger.info("Using {0} as db config source".format(os_conf)) + config.read(os_conf) + logger.info("Trying to load db config from {0}:{1}".format( + os_conf_section, os_conf_key)) + user_db_conn = config.get(os_conf_section, os_conf_key) + logger.info("Got config from {0}".format(os_conf)) + except: + logger.critical("Tried to load config from {0} but failed.".format(os_conf)) + raise +elif "DB_CONNECTION" in os.environ: + user_db_conn = os.environ['DB_CONNECTION'] + logger.info('Got config from DB_CONNECTION env var') +else: + logger.critical('Could not get db config, either from config file or env var') + sys.exit(1) + +# Root DB engine +try: + root_engine_full = create_engine(db_connection) + root_user = root_engine_full.url.username + root_password = root_engine_full.url.password + drivername = root_engine_full.url.drivername + host = root_engine_full.url.host + port = root_engine_full.url.port + root_engine_url = ''.join([drivername, '://', root_user, ':', root_password, '@', host, ':', str (port)]) + root_engine = create_engine(root_engine_url, connect_args=ssl_args) + connection = root_engine.connect() + connection.close() + logger.info("Tested connection to DB @ {0}:{1} as {2}".format( + host, port, root_user)) +except: + logger.critical('Could not connect to database as root user') + raise + +# User DB engine +try: + user_engine = create_engine(user_db_conn, connect_args=ssl_args) + # Get our user data out of the user_engine + database = user_engine.url.database + user = user_engine.url.username + password = user_engine.url.password + logger.info('Got user db config') +except: + logger.critical('Could not get user database config') + raise + +# Delete DB +try: + with root_engine.connect() as connection: + connection.execute(text("DROP DATABASE IF EXISTS {0}".format(database))) + try: + connection.commit() + except AttributeError: + pass + logger.info("Deleted database {0}".format(database)) +except: + logger.critical("Could not drop database {0}".format(database)) + raise + +# Delete DB User +try: + with root_engine.connect() as connection: + connection.execute(text("DROP USER IF EXISTS {0}".format(user))) + try: + connection.commit() + except AttributeError: + pass + logger.info("Deleted user {0}".format(user)) +except: + logger.critical("Could not delete user {0}".format(user)) + raise + +logger.info('Finished DB Management') +{{- end }} diff --git a/helm-toolkit/templates/scripts/_db-init.py.tpl b/helm-toolkit/templates/scripts/_db-init.py.tpl new file mode 100644 index 0000000000..35d04d886d --- /dev/null +++ b/helm-toolkit/templates/scripts/_db-init.py.tpl @@ -0,0 +1,167 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "helm-toolkit.scripts.db_init" }} +#!/usr/bin/env python + +# Creates db and user for an OpenStack Service: +# Set ROOT_DB_CONNECTION and DB_CONNECTION environment variables to contain +# SQLAlchemy strings for the root connection to the database and the one you +# wish the service to use. Alternatively, you can use an ini formatted config +# at the location specified by OPENSTACK_CONFIG_FILE, and extract the string +# from the key OPENSTACK_CONFIG_DB_KEY, in the section specified by +# OPENSTACK_CONFIG_DB_SECTION. + +import os +import sys +try: + import ConfigParser + PARSER_OPTS = {} +except ImportError: + import configparser as ConfigParser + PARSER_OPTS = {"strict": False} +import logging +from sqlalchemy import create_engine +from sqlalchemy import text + +# Create logger, console handler and formatter +logger = logging.getLogger('OpenStack-Helm DB Init') +logger.setLevel(logging.DEBUG) +ch = logging.StreamHandler() +ch.setLevel(logging.DEBUG) +formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') + +# Set the formatter and add the handler +ch.setFormatter(formatter) +logger.addHandler(ch) + + +# Get the connection string for the service db root user +if "ROOT_DB_CONNECTION" in os.environ: + db_connection = os.environ['ROOT_DB_CONNECTION'] + logger.info('Got DB root connection') +else: + logger.critical('environment variable ROOT_DB_CONNECTION not set') + sys.exit(1) + +mysql_x509 = os.getenv('MARIADB_X509', "") +ssl_args = {} +if mysql_x509: + ssl_args = {'ssl': {'ca': '/etc/mysql/certs/ca.crt', + 'key': '/etc/mysql/certs/tls.key', + 'cert': '/etc/mysql/certs/tls.crt'}} + +# Get the connection string for the service db +if "OPENSTACK_CONFIG_FILE" in os.environ: + os_conf = os.environ['OPENSTACK_CONFIG_FILE'] + if "OPENSTACK_CONFIG_DB_SECTION" in os.environ: + os_conf_section = os.environ['OPENSTACK_CONFIG_DB_SECTION'] + else: + logger.critical('environment variable OPENSTACK_CONFIG_DB_SECTION not set') + sys.exit(1) + if "OPENSTACK_CONFIG_DB_KEY" in os.environ: + os_conf_key = os.environ['OPENSTACK_CONFIG_DB_KEY'] + else: + logger.critical('environment variable OPENSTACK_CONFIG_DB_KEY not set') + sys.exit(1) + try: + config = ConfigParser.RawConfigParser(**PARSER_OPTS) + logger.info("Using {0} as db config source".format(os_conf)) + config.read(os_conf) + logger.info("Trying to load db config from {0}:{1}".format( + os_conf_section, os_conf_key)) + user_db_conn = config.get(os_conf_section, os_conf_key) + logger.info("Got config from {0}".format(os_conf)) + except: + logger.critical("Tried to load config from {0} but failed.".format(os_conf)) + raise +elif "DB_CONNECTION" in os.environ: + user_db_conn = os.environ['DB_CONNECTION'] + logger.info('Got config from DB_CONNECTION env var') +else: + logger.critical('Could not get db config, either from config file or env var') + sys.exit(1) + +# Root DB engine +try: + root_engine_full = create_engine(db_connection) + root_user = root_engine_full.url.username + root_password = root_engine_full.url.password + drivername = root_engine_full.url.drivername + host = root_engine_full.url.host + port = root_engine_full.url.port + root_engine_url = ''.join([drivername, '://', root_user, ':', root_password, '@', host, ':', str (port)]) + root_engine = create_engine(root_engine_url, connect_args=ssl_args) + connection = root_engine.connect() + connection.close() + logger.info("Tested connection to DB @ {0}:{1} as {2}".format( + host, port, root_user)) +except: + logger.critical('Could not connect to database as root user') + raise + +# User DB engine +try: + user_engine = create_engine(user_db_conn, connect_args=ssl_args) + # Get our user data out of the user_engine + database = user_engine.url.database + user = user_engine.url.username + password = user_engine.url.password + logger.info('Got user db config') +except: + logger.critical('Could not get user database config') + raise + +# Create DB +try: + with root_engine.connect() as connection: + connection.execute(text("CREATE DATABASE IF NOT EXISTS {0}".format(database))) + try: + connection.commit() + except AttributeError: + pass + logger.info("Created database {0}".format(database)) +except: + logger.critical("Could not create database {0}".format(database)) + raise + +# Create DB User +try: + with root_engine.connect() as connection: + connection.execute( + text("CREATE USER IF NOT EXISTS \'{0}\'@\'%\' IDENTIFIED BY \'{1}\' {2}".format( + user, password, mysql_x509))) + connection.execute( + text("GRANT ALL ON `{0}`.* TO \'{1}\'@\'%\'".format(database, user))) + try: + connection.commit() + except AttributeError: + pass + logger.info("Created user {0} for {1}".format(user, database)) +except: + logger.critical("Could not create user {0} for {1}".format(user, database)) + raise + +# Test connection +try: + connection = user_engine.connect() + connection.close() + logger.info("Tested connection to DB @ {0}:{1}/{2} as {3}".format( + host, port, database, user)) +except: + logger.critical('Could not connect to database as user') + raise + +logger.info('Finished DB Management') +{{- end }} diff --git a/helm-toolkit/templates/scripts/_db-pg-init.sh.tpl b/helm-toolkit/templates/scripts/_db-pg-init.sh.tpl new file mode 100644 index 0000000000..4d7dfaa378 --- /dev/null +++ b/helm-toolkit/templates/scripts/_db-pg-init.sh.tpl @@ -0,0 +1,69 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +{{- define "helm-toolkit.scripts.pg_db_init" }} +#!/bin/bash +set -ex + +if [[ ! -v DB_HOST ]]; then + echo "environment variable DB_HOST not set" + exit 1 +elif [[ ! -v DB_ADMIN_USER ]]; then + echo "environment variable DB_ADMIN_USER not set" + exit 1 +elif [[ ! -v PGPASSWORD ]]; then + echo "environment variable PGPASSWORD not set" + exit 1 +elif [[ ! -v DB_PORT ]]; then + echo "environment variable DB_PORT not set" + exit 1 +elif [[ ! -v USER_DB_USER ]]; then + echo "environment variable USER_DB_USER not set" + exit 1 +elif [[ ! -v USER_DB_PASS ]]; then + echo "environment variable USER_DB_PASS not set" + exit 1 +elif [[ ! -v USER_DB_NAME ]]; then + echo "environment variable USER_DB_NAME not set" + exit 1 +else + echo "Got DB connection info" +fi + +pgsql_superuser_cmd () { + DB_COMMAND="$1" + if [[ ! -z $2 ]]; then + EXPORT PGDATABASE=$2 + fi + /usr/bin/psql \ + -h ${DB_HOST} \ + -p ${DB_PORT} \ + -U ${DB_ADMIN_USER} \ + --command="${DB_COMMAND}" +} + +#create db +pgsql_superuser_cmd "SELECT 1 FROM pg_database WHERE datname = '$USER_DB_NAME'" | grep -q "(1 row)" || pgsql_superuser_cmd "CREATE DATABASE $USER_DB_NAME" + +#create db user +pgsql_superuser_cmd "SELECT * FROM pg_roles WHERE rolname = '$USER_DB_USER';" | grep -q "(1 row)" || \ + pgsql_superuser_cmd "CREATE ROLE ${USER_DB_USER} LOGIN PASSWORD '$USER_DB_PASS';" + +#Set password everytime. This is required for cases when we would want password rotation to take effect and set the updated password for a user. +pgsql_superuser_cmd "SELECT * FROM pg_roles WHERE rolname = '$USER_DB_USER';" && pgsql_superuser_cmd "ALTER USER ${USER_DB_USER} with password '$USER_DB_PASS'" + +#give permissions to user +pgsql_superuser_cmd "GRANT ALL PRIVILEGES ON DATABASE $USER_DB_NAME to $USER_DB_USER;" + +#revoke all privileges from PUBLIC role +pgsql_superuser_cmd "REVOKE ALL ON DATABASE $USER_DB_NAME FROM PUBLIC;" +{{- end }} diff --git a/helm-toolkit/templates/scripts/_image-repo-sync.sh.tpl b/helm-toolkit/templates/scripts/_image-repo-sync.sh.tpl new file mode 100644 index 0000000000..e41abe3275 --- /dev/null +++ b/helm-toolkit/templates/scripts/_image-repo-sync.sh.tpl @@ -0,0 +1,24 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "helm-toolkit.scripts.image_repo_sync" }} +#!/bin/sh +set -ex + +IFS=','; for IMAGE in ${IMAGE_SYNC_LIST}; do + docker pull ${IMAGE} + docker tag ${IMAGE} ${LOCAL_REPO}/${IMAGE} + docker push ${LOCAL_REPO}/${IMAGE} +done +{{- end }} diff --git a/helm-toolkit/templates/scripts/_ks-domain-user.sh.tpl b/helm-toolkit/templates/scripts/_ks-domain-user.sh.tpl new file mode 100644 index 0000000000..8755cd5f34 --- /dev/null +++ b/helm-toolkit/templates/scripts/_ks-domain-user.sh.tpl @@ -0,0 +1,72 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "helm-toolkit.scripts.keystone_domain_user" }} +#!/bin/bash + +# Copyright 2017 Pete Birley +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# Manage domain +SERVICE_OS_DOMAIN_ID=$(openstack domain create --or-show --enable -f value -c id \ + --description="Service Domain for ${SERVICE_OS_REGION_NAME}/${SERVICE_OS_DOMAIN_NAME}" \ + "${SERVICE_OS_DOMAIN_NAME}") + +# Display domain +openstack domain show "${SERVICE_OS_DOMAIN_ID}" + +# Manage user +SERVICE_OS_USERID=$(openstack user create --or-show --enable -f value -c id \ + --domain="${SERVICE_OS_DOMAIN_ID}" \ + --description "Service User for ${SERVICE_OS_REGION_NAME}/${SERVICE_OS_DOMAIN_NAME}" \ + --password="${SERVICE_OS_PASSWORD}" \ + "${SERVICE_OS_USERNAME}") + +# Manage user password (we do this to ensure the password is updated if required) +openstack user set --password="${SERVICE_OS_PASSWORD}" "${SERVICE_OS_USERID}" + +# Display user +openstack user show "${SERVICE_OS_USERID}" + +# Manage role +SERVICE_OS_ROLE_ID=$(openstack role show -f value -c id \ + "${SERVICE_OS_ROLE}" || openstack role create -f value -c id \ + "${SERVICE_OS_ROLE}" ) + +# Manage user role assignment +openstack role add \ + --domain="${SERVICE_OS_DOMAIN_ID}" \ + --user="${SERVICE_OS_USERID}" \ + --user-domain="${SERVICE_OS_DOMAIN_ID}" \ + "${SERVICE_OS_ROLE_ID}" + +# Display user role assignment +openstack role assignment list \ + --role="${SERVICE_OS_ROLE_ID}" \ + --user-domain="${SERVICE_OS_DOMAIN_ID}" \ + --user="${SERVICE_OS_USERID}" +{{- end }} diff --git a/helm-toolkit/templates/scripts/_ks-endpoints.sh.tpl b/helm-toolkit/templates/scripts/_ks-endpoints.sh.tpl new file mode 100755 index 0000000000..e400bcd55d --- /dev/null +++ b/helm-toolkit/templates/scripts/_ks-endpoints.sh.tpl @@ -0,0 +1,79 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "helm-toolkit.scripts.keystone_endpoints" }} +#!/bin/bash + +# Copyright 2017 Pete Birley +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# Get Service ID +OS_SERVICE_ID=$( openstack service list -f csv --quote none | \ + grep ",${OS_SERVICE_NAME},${OS_SERVICE_TYPE}$" | \ + sed -e "s/,${OS_SERVICE_NAME},${OS_SERVICE_TYPE}//g" ) + +# Get Endpoint ID if it exists +OS_ENDPOINT_ID=$( openstack endpoint list -f csv --quote none | \ + grep "^[a-z0-9]*,${OS_REGION_NAME},${OS_SERVICE_NAME},${OS_SERVICE_TYPE},True,${OS_SVC_ENDPOINT}," | \ + awk -F ',' '{ print $1 }' ) + +# Making sure only a single endpoint exists for a service within a region +if [ "$(echo $OS_ENDPOINT_ID | wc -w)" -gt "1" ]; then + echo "More than one endpoint found, cleaning up" + for ENDPOINT_ID in $OS_ENDPOINT_ID; do + openstack endpoint delete ${ENDPOINT_ID} + done + unset OS_ENDPOINT_ID +fi + +# Determine if Endpoint needs updated +if [[ ${OS_ENDPOINT_ID} ]]; then + OS_ENDPOINT_URL_CURRENT=$(openstack endpoint show ${OS_ENDPOINT_ID} -f value -c url) + if [ "${OS_ENDPOINT_URL_CURRENT}" == "${OS_SERVICE_ENDPOINT}" ]; then + echo "Endpoints Match: no action required" + OS_ENDPOINT_UPDATE="False" + else + echo "Endpoints Dont Match: removing existing entries" + openstack endpoint delete ${OS_ENDPOINT_ID} + OS_ENDPOINT_UPDATE="True" + fi +else + OS_ENDPOINT_UPDATE="True" +fi + +# Update Endpoint if required +if [[ "${OS_ENDPOINT_UPDATE}" == "True" ]]; then + OS_ENDPOINT_ID=$( openstack endpoint create -f value -c id \ + --region="${OS_REGION_NAME}" \ + "${OS_SERVICE_ID}" \ + ${OS_SVC_ENDPOINT} \ + "${OS_SERVICE_ENDPOINT}" ) +fi + +# Display the Endpoint +openstack endpoint show ${OS_ENDPOINT_ID} +{{- end }} diff --git a/helm-toolkit/templates/scripts/_ks-service.sh.tpl b/helm-toolkit/templates/scripts/_ks-service.sh.tpl new file mode 100644 index 0000000000..8356b36230 --- /dev/null +++ b/helm-toolkit/templates/scripts/_ks-service.sh.tpl @@ -0,0 +1,76 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "helm-toolkit.scripts.keystone_service" }} +#!/bin/bash + +# Copyright 2017 Pete Birley +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# Service boilerplate description +OS_SERVICE_DESC="${OS_REGION_NAME}: ${OS_SERVICE_NAME} (${OS_SERVICE_TYPE}) service" + +# Get Service ID if it exists +unset OS_SERVICE_ID + +# FIXME - There seems to be an issue once in a while where the +# openstack service list fails and encounters an error message such as: +# Unable to establish connection to +# https://keystone-api.openstack.svc.cluster.local:5000/v3/auth/tokens: +# ('Connection aborted.', OSError("(104, 'ECONNRESET')",)) +# During an upgrade scenario, this would cause the OS_SERVICE_ID to be blank +# and it would attempt to create a new service when it was not needed. +# This duplciate service would sometimes be used by other services such as +# Horizon and would give an 'Invalid Service Catalog' error. +# This loop allows for a 'retry' of the openstack service list in an +# attempt to get the service list as expected if it does ecounter an error. +# This loop and recheck can be reverted once the underlying issue is addressed. + +# If OS_SERVICE_ID is blank then wait a few seconds to give it +# additional time and try again +for i in $(seq 3) +do + OS_SERVICE_ID=$( openstack service list -f csv --quote none | \ + grep ",${OS_SERVICE_NAME},${OS_SERVICE_TYPE}$" | \ + sed -e "s/,${OS_SERVICE_NAME},${OS_SERVICE_TYPE}//g" ) + + # If the service was found, go ahead and exit successfully. + if [[ -n "${OS_SERVICE_ID}" ]]; then + exit 0 + fi + + sleep 2 +done + +# If we've reached this point and a Service ID was not found, +# then create the service +OS_SERVICE_ID=$(openstack service create -f value -c id \ + --name="${OS_SERVICE_NAME}" \ + --description "${OS_SERVICE_DESC}" \ + --enable \ + "${OS_SERVICE_TYPE}") +{{- end }} diff --git a/helm-toolkit/templates/scripts/_ks-user.sh.tpl b/helm-toolkit/templates/scripts/_ks-user.sh.tpl new file mode 100644 index 0000000000..b45f798340 --- /dev/null +++ b/helm-toolkit/templates/scripts/_ks-user.sh.tpl @@ -0,0 +1,108 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "helm-toolkit.scripts.keystone_user" }} +#!/bin/bash + +# Copyright 2017 Pete Birley +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +shopt -s nocasematch + +if [[ "${SERVICE_OS_PROJECT_DOMAIN_NAME}" == "Default" ]] +then + PROJECT_DOMAIN_ID="default" +else + # Manage project domain + PROJECT_DOMAIN_ID=$(openstack domain create --or-show --enable -f value -c id \ + --description="Domain for ${SERVICE_OS_REGION_NAME}/${SERVICE_OS_PROJECT_DOMAIN_NAME}" \ + "${SERVICE_OS_PROJECT_DOMAIN_NAME}") +fi + +if [[ "${SERVICE_OS_USER_DOMAIN_NAME}" == "Default" ]] +then + USER_DOMAIN_ID="default" +else + # Manage user domain + USER_DOMAIN_ID=$(openstack domain create --or-show --enable -f value -c id \ + --description="Domain for ${SERVICE_OS_REGION_NAME}/${SERVICE_OS_USER_DOMAIN_NAME}" \ + "${SERVICE_OS_USER_DOMAIN_NAME}") +fi + +shopt -u nocasematch + +# Manage user project +USER_PROJECT_DESC="Service Project for ${SERVICE_OS_REGION_NAME}/${SERVICE_OS_PROJECT_DOMAIN_NAME}" +USER_PROJECT_ID=$(openstack project create --or-show --enable -f value -c id \ + --domain="${PROJECT_DOMAIN_ID}" \ + --description="${USER_PROJECT_DESC}" \ + "${SERVICE_OS_PROJECT_NAME}"); + +# Manage user +USER_DESC="Service User for ${SERVICE_OS_REGION_NAME}/${SERVICE_OS_USER_DOMAIN_NAME}/${SERVICE_OS_SERVICE_NAME}" +USER_ID=$(openstack user create --or-show --enable -f value -c id \ + --domain="${USER_DOMAIN_ID}" \ + --project-domain="${PROJECT_DOMAIN_ID}" \ + --project="${USER_PROJECT_ID}" \ + --description="${USER_DESC}" \ + "${SERVICE_OS_USERNAME}"); + +# Manage user password (we do this in a seperate step to ensure the password is updated if required) +set +x +echo "Setting user password via: openstack user set --password=xxxxxxx ${USER_ID}" +openstack user set --password="${SERVICE_OS_PASSWORD}" "${USER_ID}" +set -x + +function ks_assign_user_role () { + if [[ "$SERVICE_OS_ROLE" == "admin" ]] + then + USER_ROLE_ID="$SERVICE_OS_ROLE" + else + USER_ROLE_ID=$(openstack role create --or-show -f value -c id "${SERVICE_OS_ROLE}"); + fi + + # Manage user role assignment + openstack role add \ + --user="${USER_ID}" \ + --user-domain="${USER_DOMAIN_ID}" \ + --project-domain="${PROJECT_DOMAIN_ID}" \ + --project="${USER_PROJECT_ID}" \ + "${USER_ROLE_ID}" +} + +# Manage user service role +IFS=',' +for SERVICE_OS_ROLE in ${SERVICE_OS_ROLES}; do + ks_assign_user_role +done + +# Manage user member role +: ${MEMBER_OS_ROLE:="member"} +export USER_ROLE_ID=$(openstack role create --or-show -f value -c id \ + "${MEMBER_OS_ROLE}"); +ks_assign_user_role +{{- end }} diff --git a/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl b/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl new file mode 100644 index 0000000000..e6e9c6e8e8 --- /dev/null +++ b/helm-toolkit/templates/scripts/_rabbit-init.sh.tpl @@ -0,0 +1,115 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "helm-toolkit.scripts.rabbit_init" }} +#!/bin/bash +set -e +# Extract connection details +RABBIT_HOSTNAME=$(echo "${RABBITMQ_ADMIN_CONNECTION}" | \ + awk -F'[@]' '{print $2}' | \ + awk -F'[:/]' '{print $1}') +RABBIT_PORT=$(echo "${RABBITMQ_ADMIN_CONNECTION}" | \ + awk -F'[@]' '{print $2}' | \ + awk -F'[:/]' '{print $2}') + +# Extract Admin User creadential +RABBITMQ_ADMIN_USERNAME=$(echo "${RABBITMQ_ADMIN_CONNECTION}" | \ + awk -F'[@]' '{print $1}' | \ + awk -F'[//:]' '{print $4}') +RABBITMQ_ADMIN_PASSWORD=$(echo "${RABBITMQ_ADMIN_CONNECTION}" | \ + awk -F'[@]' '{print $1}' | \ + awk -F'[//:]' '{print $5}' | \ + sed 's/%/\\x/g' | \ + xargs -0 printf "%b") + +# Extract User creadential +RABBITMQ_USERNAME=$(echo "${RABBITMQ_USER_CONNECTION}" | \ + awk -F'[@]' '{print $1}' | \ + awk -F'[//:]' '{print $4}') +RABBITMQ_PASSWORD=$(echo "${RABBITMQ_USER_CONNECTION}" | \ + awk -F'[@]' '{print $1}' | \ + awk -F'[//:]' '{print $5}' | \ + sed 's/%/\\x/g' | \ + xargs -0 printf "%b") + +# Extract User vHost +RABBITMQ_VHOST=$(echo "${RABBITMQ_USER_CONNECTION}" | \ + awk -F'[@]' '{print $2}' | \ + awk -F'[:/]' '{print $3}') +# Resolve vHost to / if no value is set +RABBITMQ_VHOST="${RABBITMQ_VHOST:-/}" + +function rabbitmqadmin_cli () { + if [ -n "$RABBITMQ_X509" ] + then + rabbitmqadmin \ + --ssl \ + --ssl-disable-hostname-verification \ + --ssl-ca-cert-file="${USER_CERT_PATH}/ca.crt" \ + --ssl-cert-file="${USER_CERT_PATH}/tls.crt" \ + --ssl-key-file="${USER_CERT_PATH}/tls.key" \ + --host="${RABBIT_HOSTNAME}" \ + --port="${RABBIT_PORT}" \ + --username="${RABBITMQ_ADMIN_USERNAME}" \ + --password="${RABBITMQ_ADMIN_PASSWORD}" \ + ${@} + else + rabbitmqadmin \ + --host="${RABBIT_HOSTNAME}" \ + --port="${RABBIT_PORT}" \ + --username="${RABBITMQ_ADMIN_USERNAME}" \ + --password="${RABBITMQ_ADMIN_PASSWORD}" \ + ${@} + fi +} + +echo "Managing: User: ${RABBITMQ_USERNAME}" +rabbitmqadmin_cli \ + declare user \ + name="${RABBITMQ_USERNAME}" \ + password="${RABBITMQ_PASSWORD}" \ + tags="user" + +echo "Deleting Guest User" +rabbitmqadmin_cli \ + delete user \ + name="guest" || true + +if [ "${RABBITMQ_VHOST}" != "/" ] +then + echo "Managing: vHost: ${RABBITMQ_VHOST}" + rabbitmqadmin_cli \ + declare vhost \ + name="${RABBITMQ_VHOST}" +else + echo "Skipping root vHost declaration: vHost: ${RABBITMQ_VHOST}" +fi + +echo "Managing: Permissions: ${RABBITMQ_USERNAME} on ${RABBITMQ_VHOST}" +rabbitmqadmin_cli \ + declare permission \ + vhost="${RABBITMQ_VHOST}" \ + user="${RABBITMQ_USERNAME}" \ + configure=".*" \ + write=".*" \ + read=".*" + +if [ ! -z "$RABBITMQ_AUXILIARY_CONFIGURATION" ] +then + echo "Applying additional configuration" + echo "${RABBITMQ_AUXILIARY_CONFIGURATION}" > /tmp/rmq_definitions.json + rabbitmqadmin_cli import /tmp/rmq_definitions.json +fi + +{{- end }} diff --git a/helm-toolkit/templates/scripts/_rally_test.sh.tpl b/helm-toolkit/templates/scripts/_rally_test.sh.tpl new file mode 100644 index 0000000000..c08d320755 --- /dev/null +++ b/helm-toolkit/templates/scripts/_rally_test.sh.tpl @@ -0,0 +1,88 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "helm-toolkit.scripts.rally_test" -}} +#!/bin/bash +set -ex +{{- $rallyTests := index . 0 }} + +: "${RALLY_ENV_NAME:="openstack-helm"}" +: "${OS_INTERFACE:="public"}" +: "${RALLY_CLEANUP:="true"}" + +if [ "x$RALLY_CLEANUP" == "xtrue" ]; then + function rally_cleanup { + openstack user delete \ + --domain="${SERVICE_OS_USER_DOMAIN_NAME}" \ + "${SERVICE_OS_USERNAME}" +{{ $rallyTests.clean_up | default "" | indent 4 }} + } + trap rally_cleanup EXIT +fi + +function create_or_update_db () { + revisionResults=$(rally db revision) + if [ $revisionResults = "None" ] + then + rally db create + else + rally db upgrade + fi +} + +create_or_update_db + +cat > /tmp/rally-config.json << EOF +{ + "openstack": { + "auth_url": "${OS_AUTH_URL}", + "region_name": "${OS_REGION_NAME}", + "endpoint_type": "${OS_INTERFACE}", + "admin": { + "username": "${OS_USERNAME}", + "password": "${OS_PASSWORD}", + "user_domain_name": "${OS_USER_DOMAIN_NAME}", + "project_name": "${OS_PROJECT_NAME}", + "project_domain_name": "${OS_PROJECT_DOMAIN_NAME}" + }, + "users": [ + { + "username": "${SERVICE_OS_USERNAME}", + "password": "${SERVICE_OS_PASSWORD}", + "project_name": "${SERVICE_OS_PROJECT_NAME}", + "user_domain_name": "${SERVICE_OS_USER_DOMAIN_NAME}", + "project_domain_name": "${SERVICE_OS_PROJECT_DOMAIN_NAME}" + } + ], + "https_insecure": false, + "https_cacert": "${OS_CACERT}" + } +} +EOF +rally deployment create --file /tmp/rally-config.json --name "${RALLY_ENV_NAME}" +rm -f /tmp/rally-config.json +rally deployment use "${RALLY_ENV_NAME}" +rally deployment check +{{- if $rallyTests.run_tempest }} +rally verify create-verifier --name "${RALLY_ENV_NAME}-tempest" --type tempest +SERVICE_TYPE="$(rally deployment check | grep "${RALLY_ENV_NAME}" | awk -F \| '{print $3}' | tr -d ' ' | tr -d '\n')" +rally verify start --pattern "tempest.api.${SERVICE_TYPE}*" +rally verify delete-verifier --id "${RALLY_ENV_NAME}-tempest" --force +{{- end }} +rally task validate /etc/rally/rally_tests.yaml +rally task start /etc/rally/rally_tests.yaml +rally task sla-check +rally env cleanup +rally deployment destroy --deployment "${RALLY_ENV_NAME}" +{{- end }} diff --git a/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl b/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl new file mode 100755 index 0000000000..695cb2e477 --- /dev/null +++ b/helm-toolkit/templates/scripts/db-backup-restore/_backup_main.sh.tpl @@ -0,0 +1,701 @@ +{{- define "helm-toolkit.scripts.db-backup-restore.backup_main" }} +#!/bin/bash + +# This file contains a database backup framework which database scripts +# can use to perform a backup. The idea here is that the database-specific +# functions will be implemented by the various databases using this script +# (like mariadb, postgresql or etcd for example). The database-specific +# script will need to first "source" this file like this: +# source /tmp/backup_main.sh +# +# Then the script should call the main backup function (backup_databases): +# backup_databases [scope] +# [scope] is an optional parameter, defaulted to "all". If only one specific +# database is required to be backed up then this parameter will +# contain the name of the database; otherwise all are backed up. +# +# The framework will require the following variables to be exported: +# +# export DB_NAMESPACE Namespace where the database(s) reside +# export DB_NAME Name of the database system +# export LOCAL_DAYS_TO_KEEP Number of days to keep the local backups +# export REMOTE_DAYS_TO_KEEP Number of days to keep the remote backups +# export ARCHIVE_DIR Local location where the backup tarballs should +# be stored. (full directory path) +# export BACK_UP_MODE Determines the mode of backup taken. +# export REMOTE_BACKUP_ENABLED "true" if remote backup enabled; false +# otherwise +# export CONTAINER_NAME Name of the container on the RGW to store +# the backup tarball. +# export STORAGE_POLICY Name of the storage policy defined on the +# RGW which is intended to store backups. +# RGW access variables: +# export OS_REGION_NAME Name of the region the RGW resides in +# export OS_AUTH_URL Keystone URL associated with the RGW +# export OS_PROJECT_NAME Name of the project associated with the +# keystone user +# export OS_USERNAME Name of the keystone user +# export OS_PASSWORD Password of the keystone user +# export OS_USER_DOMAIN_NAME Keystone domain the project belongs to +# export OS_PROJECT_DOMAIN_NAME Keystone domain the user belongs to +# export OS_IDENTITY_API_VERSION Keystone API version to use +# +# export REMOTE_BACKUP_RETRIES Number of retries to send backup to remote +# in case of any temporary failures. +# export MIN_DELAY_SEND_REMOTE Minimum seconds to delay before sending backup +# to remote to stagger backups being sent to RGW +# export MAX_DELAY_SEND_REMOTE Maximum seconds to delay before sending backup +# to remote to stagger backups being sent to RGW. +# A random number between min and max delay is generated +# to set the delay. +# +# RGW backup throttle limits variables: +# export THROTTLE_BACKUPS_ENABLED Boolean variableto control backup functionality +# export THROTTLE_LIMIT Number of simultaneous RGW upload sessions +# export THROTTLE_LOCK_EXPIRE_AFTER Time in seconds to expire flag file is orphaned +# export THROTTLE_RETRY_AFTER Time in seconds to wait before retry +# export THROTTLE_CONTAINER_NAME Name of RGW container to place flag falies into +# +# The database-specific functions that need to be implemented are: +# dump_databases_to_directory [scope] +# where: +# is the full directory path to dump the database files +# into. This is a temporary directory for this backup only. +# is the full directory path where error logs are to be +# written by the application. +# [scope] set to "all" if all databases are to be backed up; or +# set to the name of a specific database to be backed up. +# This optional parameter is defaulted to "all". +# returns: 0 if no errors; 1 if any errors occurred +# +# This function is expected to dump the database file(s) to the specified +# directory path. If this function completes successfully (returns 0), the +# framework will automatically tar/zip the files in that directory and +# name the tarball appropriately according to the proper conventions. +# +# verify_databases_backup_archives [scope] +# returns: 0 if no errors; 1 if any errors occurred +# +# This function is expected to verify the database backup archives. If this function +# completes successfully (returns 0), the +# framework will automatically starts remote backup upload. +# +# +# The functions in this file will take care of: +# 1) Calling "dump_databases_to_directory" and then compressing the files, +# naming the tarball properly, and then storing it locally at the specified +# local directory. +# 2) Sending the tarball built to the remote gateway, to be stored in the +# container configured to store database backups. +# 3) Removing local backup tarballs which are older than the number of days +# specified by the "LOCAL_DAYS_TO_KEEP" variable. +# 4) Removing remote backup tarballs (from the remote gateway) which are older +# than the number of days specified by the "REMOTE_DAYS_TO_KEEP" variable. +# 5) Controlling remote storage gateway load from client side and throttling it +# by using a dedicated RGW container to store flag files defining upload session +# in progress +# +# Note: not using set -e in this script because more elaborate error handling +# is needed. + +log_backup_error_exit() { + MSG=$1 + ERRCODE=${2:-0} + log ERROR "${DB_NAME}_backup" "${DB_NAMESPACE} namespace: ${MSG}" + rm -f $ERR_LOG_FILE + rm -rf $TMP_DIR + exit 0 +} + +log_verify_backup_exit() { + MSG=$1 + ERRCODE=${2:-0} + log ERROR "${DB_NAME}_verify_backup" "${DB_NAMESPACE} namespace: ${MSG}" + rm -f $ERR_LOG_FILE + # rm -rf $TMP_DIR + exit 0 +} + + +log() { + #Log message to a file or stdout + #TODO: This can be convert into mail alert of alert send to a monitoring system + #Params: $1 log level + #Params: $2 service + #Params: $3 message + #Params: $4 Destination + LEVEL=$1 + SERVICE=$2 + MSG=$3 + DEST=$4 + DATE=$(date +"%m-%d-%y %H:%M:%S") + if [[ -z "$DEST" ]]; then + echo "${DATE} ${LEVEL}: $(hostname) ${SERVICE}: ${MSG}" + else + echo "${DATE} ${LEVEL}: $(hostname) ${SERVICE}: ${MSG}" >>$DEST + fi +} + +# Generate a random number between MIN_DELAY_SEND_REMOTE and +# MAX_DELAY_SEND_REMOTE +random_number() { + diff=$((${MAX_DELAY_SEND_REMOTE} - ${MIN_DELAY_SEND_REMOTE} + 1)) + echo $(($(( ${RANDOM} % ${diff} )) + ${MIN_DELAY_SEND_REMOTE} )) +} + +#Get the day delta since the archive file backup +seconds_difference() { + ARCHIVE_DATE=$( date --date="$1" +%s ) + if [[ $? -ne 0 ]]; then + SECOND_DELTA=0 + fi + CURRENT_DATE=$( date +%s ) + SECOND_DELTA=$(($CURRENT_DATE-$ARCHIVE_DATE)) + if [[ "$SECOND_DELTA" -lt 0 ]]; then + SECOND_DELTA=0 + fi + echo $SECOND_DELTA +} + +# Send the specified tarball file at the specified filepath to the +# remote gateway. +send_to_remote_server() { + FILEPATH=$1 + FILE=$2 + + # Grab the list of containers on the remote site + RESULT=$(openstack container list 2>&1) + + if [[ $? -eq 0 ]]; then + echo $RESULT | grep $CONTAINER_NAME + if [[ $? -ne 0 ]]; then + # Find the swift URL from the keystone endpoint list + SWIFT_URL=$(openstack catalog show object-store -c endpoints | grep public | awk '{print $4}') + if [[ $? -ne 0 ]]; then + log WARN "${DB_NAME}_backup" "Unable to get object-store enpoints from keystone catalog." + return 2 + fi + + # Get a token from keystone + TOKEN=$(openstack token issue -f value -c id) + if [[ $? -ne 0 ]]; then + log WARN "${DB_NAME}_backup" "Unable to get keystone token." + return 2 + fi + + # Create the container + RES_FILE=$(mktemp -p /tmp) + curl -g -i -X PUT ${SWIFT_URL}/${CONTAINER_NAME} \ + -H "X-Auth-Token: ${TOKEN}" \ + -H "X-Storage-Policy: ${STORAGE_POLICY}" 2>&1 > $RES_FILE + + if [[ $? -ne 0 || $(grep "HTTP" $RES_FILE | awk '{print $2}') -ge 400 ]]; then + log WARN "${DB_NAME}_backup" "Unable to create container ${CONTAINER_NAME}" + cat $RES_FILE + rm -f $RES_FILE + return 2 + fi + rm -f $RES_FILE + + swift stat $CONTAINER_NAME + if [[ $? -ne 0 ]]; then + log WARN "${DB_NAME}_backup" "Unable to retrieve container ${CONTAINER_NAME} details after creation." + return 2 + fi + fi + else + echo $RESULT | grep -E "HTTP 401|HTTP 403" + if [[ $? -eq 0 ]]; then + log ERROR "${DB_NAME}_backup" "Access denied by keystone: ${RESULT}" + return 1 + else + echo $RESULT | grep -E "ConnectionError|Failed to discover available identity versions|Service Unavailable|HTTP 50" + if [[ $? -eq 0 ]]; then + log WARN "${DB_NAME}_backup" "Could not reach the RGW: ${RESULT}" + # In this case, keystone or the site/node may be temporarily down. + # Return slightly different error code so the calling code can retry + return 2 + else + log ERROR "${DB_NAME}_backup" "Could not get container list: ${RESULT}" + return 1 + fi + fi + fi + + # load balance delay + DELAY=$((1 + ${RANDOM} % 30)) + echo "Sleeping for ${DELAY} seconds to spread the load in time..." + sleep ${DELAY} + + #--------------------------------------------------------------------------- + # Remote backup throttling + export THROTTLE_BACKUPS_ENABLED=$(echo $THROTTLE_BACKUPS_ENABLED | sed 's/"//g') + if $THROTTLE_BACKUPS_ENABLED; then + # Remove Quotes from the constants which were added due to reading + # from secret. + export THROTTLE_LIMIT=$(echo $THROTTLE_LIMIT | sed 's/"//g') + export THROTTLE_LOCK_EXPIRE_AFTER=$(echo $THROTTLE_LOCK_EXPIRE_AFTER | sed 's/"//g') + export THROTTLE_RETRY_AFTER=$(echo $THROTTLE_RETRY_AFTER | sed 's/"//g') + export THROTTLE_CONTAINER_NAME=$(echo $THROTTLE_CONTAINER_NAME | sed 's/"//g') + + # load balance delay + RESULT=$(openstack container list 2>&1) + + if [[ $? -eq 0 ]]; then + echo $RESULT | grep $THROTTLE_CONTAINER_NAME + if [[ $? -ne 0 ]]; then + # Find the swift URL from the keystone endpoint list + SWIFT_URL=$(openstack catalog show object-store -c endpoints | grep public | awk '{print $4}') + if [[ $? -ne 0 ]]; then + log WARN "${DB_NAME}_backup" "Unable to get object-store enpoints from keystone catalog." + return 2 + fi + + # Get a token from keystone + TOKEN=$(openstack token issue -f value -c id) + if [[ $? -ne 0 ]]; then + log WARN "${DB_NAME}_backup" "Unable to get keystone token." + return 2 + fi + + # Create the container + RES_FILE=$(mktemp -p /tmp) + curl -g -i -X PUT ${SWIFT_URL}/${THROTTLE_CONTAINER_NAME} \ + -H "X-Auth-Token: ${TOKEN}" \ + -H "X-Storage-Policy: ${STORAGE_POLICY}" 2>&1 > $RES_FILE + + if [[ $? -ne 0 || $(grep "HTTP" $RES_FILE | awk '{print $2}') -ge 400 ]]; then + log WARN "${DB_NAME}_backup" "Unable to create container ${THROTTLE_CONTAINER_NAME}" + cat $RES_FILE + rm -f $RES_FILE + return 2 + fi + rm -f $RES_FILE + + swift stat $THROTTLE_CONTAINER_NAME + if [[ $? -ne 0 ]]; then + log WARN "${DB_NAME}_backup" "Unable to retrieve container ${THROTTLE_CONTAINER_NAME} details after creation." + return 2 + fi + fi + else + echo $RESULT | grep -E "HTTP 401|HTTP 403" + if [[ $? -eq 0 ]]; then + log ERROR "${DB_NAME}_backup" "Access denied by keystone: ${RESULT}" + return 1 + else + echo $RESULT | grep -E "ConnectionError|Failed to discover available identity versions|Service Unavailable|HTTP 50" + if [[ $? -eq 0 ]]; then + log WARN "${DB_NAME}_backup" "Could not reach the RGW: ${RESULT}" + # In this case, keystone or the site/node may be temporarily down. + # Return slightly different error code so the calling code can retry + return 2 + else + log ERROR "${DB_NAME}_backup" "Could not get container list: ${RESULT}" + return 1 + fi + fi + fi + + NUMBER_OF_SESSIONS=$(openstack object list $THROTTLE_CONTAINER_NAME -f value | wc -l) + log INFO "${DB_NAME}_backup" "There are ${NUMBER_OF_SESSIONS} remote sessions right now." + while [[ ${NUMBER_OF_SESSIONS} -ge ${THROTTLE_LIMIT} ]] + do + log INFO "${DB_NAME}_backup" "Current number of active uploads is ${NUMBER_OF_SESSIONS}>=${THROTTLE_LIMIT}!" + log INFO "${DB_NAME}_backup" "Retrying in ${THROTTLE_RETRY_AFTER} seconds...." + sleep ${THROTTLE_RETRY_AFTER} + NUMBER_OF_SESSIONS=$(openstack object list $THROTTLE_CONTAINER_NAME -f value | wc -l) + log INFO "${DB_NAME}_backup" "There are ${NUMBER_OF_SESSIONS} remote sessions right now." + done + + # Create a lock file in THROTTLE_CONTAINER + THROTTLE_FILEPATH=$(mktemp -d) + THROTTLE_FILE=${CONTAINER_NAME}.lock + date +%s > $THROTTLE_FILEPATH/$THROTTLE_FILE + + # Create an object to store the file + openstack object create --name $THROTTLE_FILE $THROTTLE_CONTAINER_NAME $THROTTLE_FILEPATH/$THROTTLE_FILE + if [[ $? -ne 0 ]]; then + log WARN "${DB_NAME}_backup" "Cannot create throttle container object ${THROTTLE_FILE}!" + return 2 + fi + + swift post $THROTTLE_CONTAINER_NAME $THROTTLE_FILE -H "X-Delete-After:${THROTTLE_LOCK_EXPIRE_AFTER}" + if [[ $? -ne 0 ]]; then + log WARN "${DB_NAME}_backup" "Cannot set throttle container object ${THROTTLE_FILE} expiration header!" + return 2 + fi + openstack object show $THROTTLE_CONTAINER_NAME $THROTTLE_FILE + if [[ $? -ne 0 ]]; then + log WARN "${DB_NAME}_backup" "Unable to retrieve throttle container object $THROTTLE_FILE after creation." + return 2 + fi + fi + + #--------------------------------------------------------------------------- + + # Create an object to store the file + openstack object create --name $FILE $CONTAINER_NAME $FILEPATH/$FILE + if [[ $? -ne 0 ]]; then + log WARN "${DB_NAME}_backup" "Cannot create container object ${FILE}!" + return 2 + fi + + openstack object show $CONTAINER_NAME $FILE + if [[ $? -ne 0 ]]; then + log WARN "${DB_NAME}_backup" "Unable to retrieve container object $FILE after creation." + return 2 + fi + + # Remote backup verification + MD5_REMOTE=$(openstack object show $CONTAINER_NAME $FILE -f json | jq -r ".etag") + MD5_LOCAL=$(cat ${FILEPATH}/${FILE} | md5sum | awk '{print $1}') + log INFO "${DB_NAME}_backup" "Obtained MD5 hash for the file $FILE in container $CONTAINER_NAME." + log INFO "${DB_NAME}_backup" "Local MD5 hash is ${MD5_LOCAL}." + log INFO "${DB_NAME}_backup" "Remote MD5 hash is ${MD5_REMOTE}." + if [[ "${MD5_LOCAL}" == "${MD5_REMOTE}" ]]; then + log INFO "${DB_NAME}_backup" "The local backup & remote backup MD5 hash values are matching for file $FILE in container $CONTAINER_NAME." + else + log ERROR "${DB_NAME}_backup" "Mismatch between the local backup & remote backup MD5 hash values" + return 2 + fi + rm -f ${REMOTE_FILE} + + #--------------------------------------------------------------------------- + # Remote backup throttling + export THROTTLE_BACKUPS_ENABLED=$(echo $THROTTLE_BACKUPS_ENABLED | sed 's/"//g') + if $THROTTLE_BACKUPS_ENABLED; then + # Remove flag file + # Delete an object to remove the flag file + openstack object delete $THROTTLE_CONTAINER_NAME $THROTTLE_FILE + if [[ $? -ne 0 ]]; then + log WARN "${DB_NAME}_backup" "Cannot delete throttle container object ${THROTTLE_FILE}" + return 0 + else + log INFO "${DB_NAME}_backup" "The throttle container object ${THROTTLE_FILE} has been successfully removed." + fi + rm -f ${THROTTLE_FILEPATH}/${THROTTLE_FILE} + fi + + #--------------------------------------------------------------------------- + + log INFO "${DB_NAME}_backup" "Created file $FILE in container $CONTAINER_NAME successfully." + return 0 +} + +# This function attempts to store the built tarball to the remote gateway, +# with built-in logic to handle error cases like: +# 1) Network connectivity issues - retries for a specific amount of time +# 2) Authorization errors - immediately logs an ERROR and returns +store_backup_remotely() { + FILEPATH=$1 + FILE=$2 + + count=1 + while [[ ${count} -le ${REMOTE_BACKUP_RETRIES} ]]; do + # Store the new archive to the remote backup storage facility. + send_to_remote_server $FILEPATH $FILE + SEND_RESULT="$?" + + # Check if successful + if [[ $SEND_RESULT -eq 0 ]]; then + log INFO "${DB_NAME}_backup" "Backup file ${FILE} successfully sent to RGW." + return 0 + elif [[ $SEND_RESULT -eq 2 ]]; then + if [[ ${count} -ge ${REMOTE_BACKUP_RETRIES} ]]; then + log ERROR "${DB_NAME}_backup" "Backup file ${FILE} could not be sent to the RGW in " \ + "${REMOTE_BACKUP_RETRIES} retries. Errors encountered. Exiting." + break + fi + # Temporary failure occurred. We need to retry + log WARN "${DB_NAME}_backup" "Backup file ${FILE} could not be sent to RGW due to connection issue." + sleep_time=$(random_number) + log INFO "${DB_NAME}_backup" "Sleeping ${sleep_time} seconds waiting for RGW to become available..." + sleep ${sleep_time} + log INFO "${DB_NAME}_backup" "Retrying..." + else + log ERROR "${DB_NAME}_backup" "Backup file ${FILE} could not be sent to the RGW. Errors encountered. Exiting." + break + fi + + # Increment the counter + count=$((count+1)) + done + + return 1 +} + + +function get_archive_date(){ +# get_archive_date function returns correct archive date +# for different formats of archives' names +# the old one: ....tar.gz +# the new one: ..
...tar.gz + local A_FILE="$1" + awk -F. '{print $(NF-2)}' <<< ${A_FILE} | tr -d "Z" +} + +# This function takes a list of archives' names as an input +# and creates a hash table where keys are number of seconds +# between current date and archive date (see seconds_difference), +# and values are space separated archives' names +# +# +------------+---------------------------------------------------------------------------------------------------------+ +# | 1265342678 | "tmp/mysql.backup.auto.2022-02-14T10:13:13Z.tar.gz" | +# +------------+---------------------------------------------------------------------------------------------------------+ +# | 2346254257 | "tmp/mysql.backup.auto.2022-02-11T10:13:13Z.tar.gz tmp/mysql.backup.manual.2022-02-11T10:13:13Z.tar.gz" | +# +------------+---------------------------------------------------------------------------------------------------------+ +# <...> +# +------------+---------------------------------------------------------------------------------------------------------+ +# | 6253434567 | "tmp/mysql.backup.manual.2022-02-01T10:13:13Z.tar.gz" | +# +------------+---------------------------------------------------------------------------------------------------------+ +# We will use the explained above data stracture to cover rare, but still +# possible case, when we have several backups of the same date. E.g. +# one manual, and one automatic. + +declare -A fileTable +create_hash_table() { +unset fileTable +fileList=$@ + for ARCHIVE_FILE in ${fileList}; do + # Creating index, we will round given ARCHIVE_DATE to the midnight (00:00:00) + # to take in account a possibility, that we can have more than one scheduled + # backup per day. + ARCHIVE_DATE=$(get_archive_date ${ARCHIVE_FILE}) + ARCHIVE_DATE=$(date --date=${ARCHIVE_DATE} +%D) + log INFO "${DB_NAME}_backup" "Archive date to build index: ${ARCHIVE_DATE}" + INDEX=$(seconds_difference ${ARCHIVE_DATE}) + if [[ -z fileTable[${INDEX}] ]]; then + fileTable[${INDEX}]=${ARCHIVE_FILE} + else + fileTable[${INDEX}]="${fileTable[${INDEX}]} ${ARCHIVE_FILE}" + fi + echo "INDEX: ${INDEX} VALUE: ${fileTable[${INDEX}]}" + done +} + +function get_backup_prefix() { +# Create list of all possible prefixes in a format: +# . to cover a possible situation +# when different backups of different databases and/or +# namespaces share the same local or remote storage. + ALL_FILES=($@) + PREFIXES=() + for fname in ${ALL_FILES[@]}; do + prefix=$(basename ${fname} | cut -d'.' -f1,2 ) + for ((i=0; i<${#PREFIXES[@]}; i++)) do + if [[ ${PREFIXES[${i}]} == ${prefix} ]]; then + prefix="" + break + fi + done + if [[ ! -z ${prefix} ]]; then + PREFIXES+=(${prefix}) + fi + done +} + +remove_old_local_archives() { + SECONDS_TO_KEEP=$(( $((${LOCAL_DAYS_TO_KEEP}))*86400)) + log INFO "${DB_NAME}_backup" "Deleting backups older than ${LOCAL_DAYS_TO_KEEP} days (${SECONDS_TO_KEEP} seconds)" + if [[ -d $ARCHIVE_DIR ]]; then + count=0 + # We iterate over the hash table, checking the delta in seconds (hash keys), + # and minimum number of backups we must have in place. List of keys has to be sorted. + for INDEX in $(tr " " "\n" <<< ${!fileTable[@]} | sort -n -); do + ARCHIVE_FILE=${fileTable[${INDEX}]} + if [[ ${INDEX} -lt ${SECONDS_TO_KEEP} || ${count} -lt ${LOCAL_DAYS_TO_KEEP} ]]; then + ((count++)) + log INFO "${DB_NAME}_backup" "Keeping file(s) ${ARCHIVE_FILE}." + else + log INFO "${DB_NAME}_backup" "Deleting file(s) ${ARCHIVE_FILE}." + rm -f ${ARCHIVE_FILE} + if [[ $? -ne 0 ]]; then + # Log error but don't exit so we can finish the script + # because at this point we haven't sent backup to RGW yet + log ERROR "${DB_NAME}_backup" "Failed to cleanup local backup. Cannot remove some of ${ARCHIVE_FILE}" + fi + fi + done + else + log WARN "${DB_NAME}_backup" "The local backup directory ${$ARCHIVE_DIR} does not exist." + fi +} + +prepare_list_of_remote_backups() { + BACKUP_FILES=$(mktemp -p /tmp) + DB_BACKUP_FILES=$(mktemp -p /tmp) + openstack object list $CONTAINER_NAME > $BACKUP_FILES + if [[ $? -ne 0 ]]; then + log_backup_error_exit \ + "Failed to cleanup remote backup. Could not obtain a list of current backup files in the RGW" + fi + # Filter out other types of backup files + cat $BACKUP_FILES | grep $DB_NAME | grep $DB_NAMESPACE | awk '{print $2}' > $DB_BACKUP_FILES +} + +# The logic implemented with this function is absolutely similar +# to the function remove_old_local_archives (see above) +remove_old_remote_archives() { + count=0 + SECONDS_TO_KEEP=$((${REMOTE_DAYS_TO_KEEP}*86400)) + log INFO "${DB_NAME}_backup" "Deleting backups older than ${REMOTE_DAYS_TO_KEEP} days (${SECONDS_TO_KEEP} seconds)" + for INDEX in $(tr " " "\n" <<< ${!fileTable[@]} | sort -n -); do + ARCHIVE_FILE=${fileTable[${INDEX}]} + if [[ ${INDEX} -lt ${SECONDS_TO_KEEP} || ${count} -lt ${REMOTE_DAYS_TO_KEEP} ]]; then + ((count++)) + log INFO "${DB_NAME}_backup" "Keeping remote backup(s) ${ARCHIVE_FILE}." + else + log INFO "${DB_NAME}_backup" "Deleting remote backup(s) ${ARCHIVE_FILE} from the RGW" + openstack object delete ${CONTAINER_NAME} ${ARCHIVE_FILE} || log WARN "${DB_NAME}_backup" \ + "Failed to cleanup remote backup. Cannot delete container object ${ARCHIVE_FILE}" + fi + done + + # Cleanup now that we're done. + for fd in ${BACKUP_FILES} ${DB_BACKUP_FILES}; do + if [[ -f ${fd} ]]; then + rm -f ${fd} + else + log WARN "${DB_NAME}_backup" "Can not delete a temporary file ${fd}" + fi + done +} + +# Main function to backup the databases. Calling functions need to supply: +# 1) The directory where the final backup will be kept after it is compressed. +# 2) A temporary directory to use for placing database files to be compressed. +# Note: this temp directory will be deleted after backup is done. +# 3) Optional "scope" parameter indicating what database to back up. Defaults +# to "all". +backup_databases() { + SCOPE=${1:-"all"} + + # Create necessary directories if they do not exist. + mkdir -p $ARCHIVE_DIR || log_backup_error_exit \ + "Backup of the ${DB_NAME} database failed. Cannot create directory ${ARCHIVE_DIR}!" + export TMP_DIR=$(mktemp -d) || log_backup_error_exit \ + "Backup of the ${DB_NAME} database failed. Cannot create temp directory!" + + # Create temporary log file + export ERR_LOG_FILE=$(mktemp -p /tmp) || log_backup_error_exit \ + "Backup of the ${DB_NAME} database failed. Cannot create log file!" + + # It is expected that this function will dump the database files to the $TMP_DIR + dump_databases_to_directory $TMP_DIR $ERR_LOG_FILE $SCOPE + + # If successful, there should be at least one file in the TMP_DIR + if [[ $? -ne 0 || $(ls $TMP_DIR | wc -w) -eq 0 ]]; then + cat $ERR_LOG_FILE + log_backup_error_exit "Backup of the ${DB_NAME} database failed and needs attention." + fi + + log INFO "${DB_NAME}_backup" "Databases dumped successfully. Creating tarball..." + + NOW=$(date +"%Y-%m-%dT%H:%M:%SZ") + if [[ -z "${BACK_UP_MODE}" ]]; then + TARBALL_FILE="${DB_NAME}.${DB_NAMESPACE}.${SCOPE}.${NOW}.tar.gz" + else + TARBALL_FILE="${DB_NAME}.${DB_NAMESPACE}.${SCOPE}.${BACK_UP_MODE}.${NOW}.tar.gz" + fi + + cd $TMP_DIR || log_backup_error_exit \ + "Backup of the ${DB_NAME} database failed. Cannot change to directory $TMP_DIR" + + #Archive the current database files + tar zcvf $ARCHIVE_DIR/$TARBALL_FILE * + if [[ $? -ne 0 ]]; then + log_backup_error_exit \ + "Backup ${DB_NAME} to local file system failed. Backup tarball could not be created." + fi + + # Get the size of the file + ARCHIVE_SIZE=$(ls -l $ARCHIVE_DIR/$TARBALL_FILE | awk '{print $5}') + + log INFO "${DB_NAME}_backup" "Tarball $TARBALL_FILE created successfully." + + cd $ARCHIVE_DIR + + #Only delete the old archive after a successful archive + export LOCAL_DAYS_TO_KEEP=$(echo $LOCAL_DAYS_TO_KEEP | sed 's/"//g') + if [[ "$LOCAL_DAYS_TO_KEEP" -gt 0 ]]; then + get_backup_prefix $(ls -1 ${ARCHIVE_DIR}/*.gz) + for ((i=0; i<${#PREFIXES[@]}; i++)); do + echo "Working with prefix: ${PREFIXES[i]}" + create_hash_table $(ls -1 ${ARCHIVE_DIR}/${PREFIXES[i]}*.gz) + remove_old_local_archives + done + fi + + # Local backup verification process + + # It is expected that this function will verify the database backup files + if verify_databases_backup_archives ${SCOPE}; then + log INFO "${DB_NAME}_backup_verify" "Databases backup verified successfully. Uploading verified backups to remote location..." + else + # If successful, there should be at least one file in the TMP_DIR + if [[ $(ls $TMP_DIR | wc -w) -eq 0 ]]; then + cat $ERR_LOG_FILE + fi + log_verify_backup_exit "Verify of the ${DB_NAME} database backup failed and needs attention." + exit 1 + fi + + # Remove the temporary directory and files as they are no longer needed. + rm -rf $TMP_DIR + rm -f $ERR_LOG_FILE + + # Remote backup + REMOTE_BACKUP=$(echo $REMOTE_BACKUP_ENABLED | sed 's/"//g') + if $REMOTE_BACKUP; then + # Remove Quotes from the constants which were added due to reading + # from secret. + export REMOTE_BACKUP_RETRIES=$(echo $REMOTE_BACKUP_RETRIES | sed 's/"//g') + export MIN_DELAY_SEND_REMOTE=$(echo $MIN_DELAY_SEND_REMOTE | sed 's/"//g') + export MAX_DELAY_SEND_REMOTE=$(echo $MAX_DELAY_SEND_REMOTE | sed 's/"//g') + export REMOTE_DAYS_TO_KEEP=$(echo $REMOTE_DAYS_TO_KEEP | sed 's/"//g') + + store_backup_remotely $ARCHIVE_DIR $TARBALL_FILE + if [[ $? -ne 0 ]]; then + # This error should print first, then print the summary as the last + # thing that the user sees in the output. + log ERROR "${DB_NAME}_backup" "Backup ${TARBALL_FILE} could not be sent to remote RGW." + echo "==================================================================" + echo "Local backup successful, but could not send to remote RGW." + echo "Backup archive name: $TARBALL_FILE" + echo "Backup archive size: $ARCHIVE_SIZE" + echo "==================================================================" + # Because the local backup was successful, exit with 0 so the pod will not + # continue to restart and fill the disk with more backups. The ERRORs are + # logged and alerting system should catch those errors and flag the operator. + exit 0 + fi + + #Only delete the old archive after a successful archive + if [[ "$REMOTE_DAYS_TO_KEEP" -gt 0 ]]; then + prepare_list_of_remote_backups + get_backup_prefix $(cat $DB_BACKUP_FILES) + for ((i=0; i<${#PREFIXES[@]}; i++)); do + echo "Working with prefix: ${PREFIXES[i]}" + create_hash_table $(cat ${DB_BACKUP_FILES} | grep ${PREFIXES[i]}) + remove_old_remote_archives + done + fi + + echo "==================================================================" + echo "Local backup and backup to remote RGW successful!" + echo "Backup archive name: $TARBALL_FILE" + echo "Backup archive size: $ARCHIVE_SIZE" + echo "==================================================================" + else + # Remote backup is not enabled. This is ok; at least we have a local backup. + log INFO "${DB_NAME}_backup" "Skipping remote backup, as it is not enabled." + + echo "==================================================================" + echo "Local backup successful!" + echo "Backup archive name: $TARBALL_FILE" + echo "Backup archive size: $ARCHIVE_SIZE" + echo "==================================================================" + fi +} +{{- end }} \ No newline at end of file diff --git a/helm-toolkit/templates/scripts/db-backup-restore/_restore_main.sh.tpl b/helm-toolkit/templates/scripts/db-backup-restore/_restore_main.sh.tpl new file mode 100755 index 0000000000..093dd2cc9b --- /dev/null +++ b/helm-toolkit/templates/scripts/db-backup-restore/_restore_main.sh.tpl @@ -0,0 +1,616 @@ +{{- define "helm-toolkit.scripts.db-backup-restore.restore_main" }} +#!/bin/bash + +# This file contains a database restore framework which database scripts +# can use to perform a backup. The idea here is that the database-specific +# functions will be implemented by the various databases using this script +# (like mariadb, postgresql or etcd for example). The database-specific +# script will need to first "source" this file like this: +# source /tmp/restore_main.sh +# +# Then the script should call the main CLI function (cli_main): +# cli_main +# where: +# is the list of arguments given by the user +# +# The framework will require the following variables to be exported: +# +# export DB_NAMESPACE Namespace where the database(s) reside +# export DB_NAME Name of the database system +# export ARCHIVE_DIR Location where the backup tarballs should +# be stored. (full directory path which +# should already exist) +# export CONTAINER_NAME Name of the container on the RGW where +# the backups are stored. +# RGW access variables: +# export OS_REGION_NAME Name of the region the RGW resides in +# export OS_AUTH_URL Keystone URL associated with the RGW +# export OS_PROJECT_NAME Name of the project associated with the +# keystone user +# export OS_USERNAME Name of the keystone user +# export OS_PASSWORD Password of the keystone user +# export OS_USER_DOMAIN_NAME Keystone domain the project belongs to +# export OS_PROJECT_DOMAIN_NAME Keystone domain the user belongs to +# export OS_IDENTITY_API_VERSION Keystone API version to use +# +# The database-specific functions that need to be implemented are: +# get_databases +# where: +# is the full directory path where the decompressed +# database files reside +# is the full path of the file to write the database +# names into, one database per line +# returns: 0 if no errors; 1 if any errors occurred +# +# This function is expected to extract the database names from the +# uncompressed database files found in the given "tmp_dir", which is +# the staging directory for database restore. The database names +# should be written to the given "db_file", one database name per +# line. +# +# get_tables +# is the name of the database to get the tables from +# is the full directory path where the decompressed +# database files reside +# is the full path of the file to write the table +# names into, one table per line +# returns: 0 if no errors; 1 if any errors occurred +# +# This function is expected to extract the table names from the given +# database, found in the uncompressed database files located in the +# given "tmp_dir", which is the staging directory for database restore. +# The table names should be written to the given "table_file", one +# table name per line. +# +# get_rows +# is the name of the table to get the rows from +# is the name of the database the table resides in +# is the full directory path where the decompressed +# database files reside +# is the full path of the file to write the table +# row data into, one row (INSERT statement) per line +# returns: 0 if no errors; 1 if any errors occurred +# +# This function is expected to extract the rows from the given table +# in the given database, found in the uncompressed database files +# located in the given "tmp_dir", which is the staging directory for +# database restore. The table rows should be written to the given +# "rows_file", one row (INSERT statement) per line. +# +# get_schema +# is the name of the table to get the schema from +# is the name of the database the table resides in +# is the full directory path where the decompressed +# database files reside +# is the full path of the file to write the table +# schema data into +# returns: 0 if no errors; 1 if any errors occurred +# +# This function is expected to extract the schema from the given table +# in the given database, found in the uncompressed database files +# located in the given "tmp_dir", which is the staging directory for +# database restore. The table schema and related alterations and +# grant information should be written to the given "schema_file". +# +# restore_single_db +# where: +# is the name of the database to be restored +# is the full directory path where the decompressed +# database files reside +# returns: 0 if no errors; 1 if any errors occurred +# +# This function is expected to restore the database given as "db_name" +# using the database files located in the "tmp_dir". The framework +# will delete the "tmp_dir" and the files in it after the restore is +# complete. +# +# restore_all_dbs +# where: +# is the full directory path where the decompressed +# database files reside +# returns: 0 if no errors; 1 if any errors occurred +# +# This function is expected to restore all of the databases which +# are backed up in the database files located in the "tmp_dir". The +# framework will delete the "tmp_dir" and the files in it after the +# restore is complete. +# +# The functions in this file will take care of: +# 1) The CLI parameter parsing for the arguments passed in by the user. +# 2) The listing of either local or remote archive files at the request +# of the user. +# 3) The retrieval/download of an archive file located either in the local +# file system or remotely stored on an RGW. +# 4) Calling either "restore_single_db" or "restore_all_dbs" when the user +# chooses to restore a database or all databases. +# 5) The framework will call "get_databases" when it needs a list of +# databases when the user requests a database list or when the user +# requests to restore a single database (to ensure it exists in the +# archive). Similarly, the framework will call "get_tables", "get_rows", +# or "get_schema" when it needs that data requested by the user. +# + +usage() { + ret_val=$1 + echo "Usage:" + echo "Restore command options" + echo "=============================" + echo "help" + echo "list_archives [remote]" + echo "list_databases [remote]" + echo "list_tables [remote]" + echo "list_rows [remote]" + echo "list_schema [remote]" + echo "restore [remote]" + echo " where = | ALL" + echo "delete_archive [remote]" + clean_and_exit $ret_val "" +} + +#Exit cleanly with some message and return code +clean_and_exit() { + RETCODE=$1 + MSG=$2 + + # Clean/remove temporary directories/files + rm -rf $TMP_DIR + rm -f $RESULT_FILE + + if [[ "x${MSG}" != "x" ]]; then + echo $MSG + fi + exit $RETCODE +} + +determine_resulting_error_code() { + RESULT="$1" + + echo ${RESULT} | grep "HTTP 404" + if [[ $? -eq 0 ]]; then + echo "Could not find the archive: ${RESULT}" + return 1 + else + echo ${RESULT} | grep "HTTP 401" + if [[ $? -eq 0 ]]; then + echo "Could not access the archive: ${RESULT}" + return 1 + else + echo ${RESULT} | grep "HTTP 503" + if [[ $? -eq 0 ]]; then + echo "RGW service is unavailable. ${RESULT}" + # In this case, the RGW may be temporarily down. + # Return slightly different error code so the calling code can retry + return 2 + else + echo ${RESULT} | grep "ConnectionError" + if [[ $? -eq 0 ]]; then + echo "Could not reach the RGW: ${RESULT}" + # In this case, keystone or the site/node may be temporarily down. + # Return slightly different error code so the calling code can retry + return 2 + else + echo "Archive ${ARCHIVE} could not be retrieved: ${RESULT}" + return 1 + fi + fi + fi + fi + return 0 +} + +# Retrieve a list of archives from the RGW. +retrieve_remote_listing() { + RESULT=$(openstack container show $CONTAINER_NAME 2>&1) + if [[ $? -eq 0 ]]; then + # Get the list, ensureing that we only pick up the right kind of backups from the + # requested namespace + openstack object list $CONTAINER_NAME | grep $DB_NAME | grep $DB_NAMESPACE | awk '{print $2}' > $TMP_DIR/archive_list + if [[ $? -ne 0 ]]; then + echo "Container object listing could not be obtained." + return 1 + else + echo "Archive listing successfully retrieved." + fi + else + determine_resulting_error_code "${RESULT}" + return $? + fi + return 0 +} + +# Retrieve a single archive from the RGW. +retrieve_remote_archive() { + ARCHIVE=$1 + + RESULT=$(openstack object save --file $TMP_DIR/$ARCHIVE $CONTAINER_NAME $ARCHIVE 2>&1) + if [[ $? -ne 0 ]]; then + determine_resulting_error_code "${RESULT}" + return $? + else + echo "Archive $ARCHIVE successfully retrieved." + fi + return 0 +} + +# Delete an archive from the RGW. +delete_remote_archive() { + ARCHIVE=$1 + + RESULT=$(openstack object delete ${CONTAINER_NAME} ${ARCHIVE} 2>&1) + if [[ $? -ne 0 ]]; then + determine_resulting_error_code "${RESULT}" + return $? + else + echo "Archive ${ARCHIVE} successfully deleted." + fi + return 0 +} + +# Display all archives +list_archives() { + REMOTE=$1 + + if [[ "x${REMOTE^^}" == "xREMOTE" ]]; then + retrieve_remote_listing + if [[ $? -eq 0 && -e $TMP_DIR/archive_list ]]; then + echo + echo "All Archives from RGW Data Store" + echo "==============================================" + cat $TMP_DIR/archive_list | sort + clean_and_exit 0 "" + else + clean_and_exit 1 "ERROR: Archives could not be retrieved from the RGW." + fi + elif [[ "x${REMOTE}" == "x" ]]; then + if [[ -d $ARCHIVE_DIR ]]; then + archives=$(find $ARCHIVE_DIR/ -iname "*.gz" -print | sort) + echo + echo "All Local Archives" + echo "==============================================" + for archive in $archives + do + echo $archive | cut -d '/' -f8- + done + clean_and_exit 0 "" + else + clean_and_exit 1 "ERROR: Local archive directory is not available." + fi + else + usage 1 + fi +} + +# Retrieve the archive from the desired location and decompress it into +# the restore directory +get_archive() { + ARCHIVE_FILE=$1 + REMOTE=$2 + + if [[ "x$REMOTE" == "xremote" ]]; then + echo "Retrieving archive ${ARCHIVE_FILE} from the remote RGW..." + retrieve_remote_archive $ARCHIVE_FILE + if [[ $? -ne 0 ]]; then + clean_and_exit 1 "ERROR: Could not retrieve remote archive: $ARCHIVE_FILE" + fi + elif [[ "x$REMOTE" == "x" ]]; then + if [[ -e $ARCHIVE_DIR/$ARCHIVE_FILE ]]; then + cp $ARCHIVE_DIR/$ARCHIVE_FILE $TMP_DIR/$ARCHIVE_FILE + if [[ $? -ne 0 ]]; then + clean_and_exit 1 "ERROR: Could not copy local archive to restore directory." + fi + else + clean_and_exit 1 "ERROR: Local archive file could not be found." + fi + else + usage 1 + fi + + echo "Decompressing archive $ARCHIVE_FILE..." + cd $TMP_DIR + tar zxvf - < $TMP_DIR/$ARCHIVE_FILE 1>/dev/null + if [[ $? -ne 0 ]]; then + clean_and_exit 1 "ERROR: Archive decompression failed." + fi +} + +# Display all databases from an archive +list_databases() { + ARCHIVE_FILE=$1 + REMOTE=$2 + WHERE="local" + + if [[ -n ${REMOTE} ]]; then + WHERE="remote" + fi + + # Get the archive from the source location (local/remote) + get_archive $ARCHIVE_FILE $REMOTE + + # Expectation is that the database listing will be put into + # the given file one database per line + get_databases $TMP_DIR $RESULT_FILE + if [[ "$?" -ne 0 ]]; then + clean_and_exit 1 "ERROR: Could not retrieve databases from $WHERE archive $ARCHIVE_FILE." + fi + + if [[ -f "$RESULT_FILE" ]]; then + echo " " + echo "Databases in the $WHERE archive $ARCHIVE_FILE" + echo "================================================================================" + cat $RESULT_FILE + else + clean_and_exit 1 "ERROR: Databases file missing. Could not list databases from $WHERE archive $ARCHIVE_FILE." + fi +} + +# Display all tables of a database from an archive +list_tables() { + ARCHIVE_FILE=$1 + DATABASE=$2 + REMOTE=$3 + WHERE="local" + + if [[ -n ${REMOTE} ]]; then + WHERE="remote" + fi + + # Get the archive from the source location (local/remote) + get_archive $ARCHIVE_FILE $REMOTE + + # Expectation is that the database listing will be put into + # the given file one table per line + get_tables $DATABASE $TMP_DIR $RESULT_FILE + if [[ "$?" -ne 0 ]]; then + clean_and_exit 1 "ERROR: Could not retrieve tables for database ${DATABASE} from $WHERE archive $ARCHIVE_FILE." + fi + + if [[ -f "$RESULT_FILE" ]]; then + echo " " + echo "Tables in database $DATABASE from $WHERE archive $ARCHIVE_FILE" + echo "================================================================================" + cat $RESULT_FILE + else + clean_and_exit 1 "ERROR: Tables file missing. Could not list tables of database ${DATABASE} from $WHERE archive $ARCHIVE_FILE." + fi +} + +# Display all rows of the given database table from an archive +list_rows() { + ARCHIVE_FILE=$1 + DATABASE=$2 + TABLE=$3 + REMOTE=$4 + WHERE="local" + + if [[ -n ${REMOTE} ]]; then + WHERE="remote" + fi + + # Get the archive from the source location (local/remote) + get_archive $ARCHIVE_FILE $REMOTE + + # Expectation is that the database listing will be put into + # the given file one table per line + get_rows $DATABASE $TABLE $TMP_DIR $RESULT_FILE + if [[ "$?" -ne 0 ]]; then + clean_and_exit 1 "ERROR: Could not retrieve rows in table ${TABLE} of database ${DATABASE} from $WHERE archive $ARCHIVE_FILE." + fi + + if [[ -f "$RESULT_FILE" ]]; then + echo " " + echo "Rows in table $TABLE of database $DATABASE from $WHERE archive $ARCHIVE_FILE" + echo "================================================================================" + cat $RESULT_FILE + else + clean_and_exit 1 "ERROR: Rows file missing. Could not list rows in table ${TABLE} of database ${DATABASE} from $WHERE archive $ARCHIVE_FILE." + fi +} + +# Display the schema information of the given database table from an archive +list_schema() { + ARCHIVE_FILE=$1 + DATABASE=$2 + TABLE=$3 + REMOTE=$4 + WHERE="local" + + if [[ -n ${REMOTE} ]]; then + WHERE="remote" + fi + + # Get the archive from the source location (local/remote) + get_archive $ARCHIVE_FILE $REMOTE + + # Expectation is that the schema information will be placed into + # the given schema file. + get_schema $DATABASE $TABLE $TMP_DIR $RESULT_FILE + if [[ "$?" -ne 0 ]]; then + clean_and_exit 1 "ERROR: Could not retrieve schema for table ${TABLE} of database ${DATABASE} from $WHERE archive $ARCHIVE_FILE." + fi + + if [[ -f "$RESULT_FILE" ]]; then + echo " " + echo "Schema for table $TABLE of database $DATABASE from $WHERE archive $ARCHIVE_FILE" + echo "================================================================================" + cat $RESULT_FILE + else + clean_and_exit 1 "ERROR: Schema file missing. Could not list schema for table ${TABLE} of database ${DATABASE} from $WHERE archive $ARCHIVE_FILE." + fi +} + +# Delete an archive +delete_archive() { + ARCHIVE_FILE=$1 + REMOTE=$2 + WHERE="local" + + if [[ -n ${REMOTE} ]]; then + WHERE="remote" + fi + + if [[ "${WHERE}" == "remote" ]]; then + delete_remote_archive ${ARCHIVE_FILE} + if [[ $? -ne 0 ]]; then + clean_and_exit 1 "ERROR: Could not delete remote archive: ${ARCHIVE_FILE}" + fi + else # Local + if [[ -e ${ARCHIVE_DIR}/${ARCHIVE_FILE} ]]; then + rm -f ${ARCHIVE_DIR}/${ARCHIVE_FILE} + if [[ $? -ne 0 ]]; then + clean_and_exit 1 "ERROR: Could not delete local archive." + fi + else + clean_and_exit 1 "ERROR: Local archive file could not be found." + fi + fi + + echo "Successfully deleted archive ${ARCHIVE_FILE} from ${WHERE} storage." +} + + +# Return 1 if the given database exists in the database file. 0 otherwise. +database_exists() { + DB=$1 + + grep "${DB}" ${RESULT_FILE} + if [[ $? -eq 0 ]]; then + return 1 + fi + return 0 +} + +# This is the main CLI interpreter function +cli_main() { + ARGS=("$@") + + # Create the ARCHIVE DIR if it's not already there. + mkdir -p $ARCHIVE_DIR + + # Create temp directory for a staging area to decompress files into + export TMP_DIR=$(mktemp -d) + + # Create a temp file for storing list of databases (if needed) + export RESULT_FILE=$(mktemp -p /tmp) + + case "${ARGS[0]}" in + "help") + usage 0 + ;; + + "list_archives") + if [[ ${#ARGS[@]} -gt 2 ]]; then + usage 1 + elif [[ ${#ARGS[@]} -eq 1 ]]; then + list_archives + else + list_archives ${ARGS[1]} + fi + clean_and_exit 0 + ;; + + "list_databases") + if [[ ${#ARGS[@]} -lt 2 || ${#ARGS[@]} -gt 3 ]]; then + usage 1 + elif [[ ${#ARGS[@]} -eq 2 ]]; then + list_databases ${ARGS[1]} + else + list_databases ${ARGS[1]} ${ARGS[2]} + fi + ;; + + "list_tables") + if [[ ${#ARGS[@]} -lt 3 || ${#ARGS[@]} -gt 4 ]]; then + usage 1 + elif [[ ${#ARGS[@]} -eq 3 ]]; then + list_tables ${ARGS[1]} ${ARGS[2]} + else + list_tables ${ARGS[1]} ${ARGS[2]} ${ARGS[3]} + fi + ;; + + "list_rows") + if [[ ${#ARGS[@]} -lt 4 || ${#ARGS[@]} -gt 5 ]]; then + usage 1 + elif [[ ${#ARGS[@]} -eq 4 ]]; then + list_rows ${ARGS[1]} ${ARGS[2]} ${ARGS[3]} + else + list_rows ${ARGS[1]} ${ARGS[2]} ${ARGS[3]} ${ARGS[4]} + fi + ;; + + "list_schema") + if [[ ${#ARGS[@]} -lt 4 || ${#ARGS[@]} -gt 5 ]]; then + usage 1 + elif [[ ${#ARGS[@]} -eq 4 ]]; then + list_schema ${ARGS[1]} ${ARGS[2]} ${ARGS[3]} + else + list_schema ${ARGS[1]} ${ARGS[2]} ${ARGS[3]} ${ARGS[4]} + fi + ;; + + "restore") + REMOTE="" + if [[ ${#ARGS[@]} -lt 3 || ${#ARGS[@]} -gt 4 ]]; then + usage 1 + elif [[ ${#ARGS[@]} -eq 4 ]]; then + REMOTE=${ARGS[3]} + fi + + ARCHIVE=${ARGS[1]} + DB_SPEC=${ARGS[2]} + + #Get all the databases in that archive + get_archive $ARCHIVE $REMOTE + + if [[ "$( echo $DB_SPEC | tr '[a-z]' '[A-Z]')" != "ALL" ]]; then + # Expectation is that the database listing will be put into + # the given file one database per line + get_databases $TMP_DIR $RESULT_FILE + if [[ "$?" -ne 0 ]]; then + clean_and_exit 1 "ERROR: Could not get the list of databases to restore." + fi + + if [[ ! $DB_NAMESPACE == "kube-system" ]]; then + #check if the requested database is available in the archive + database_exists $DB_SPEC + if [[ $? -ne 1 ]]; then + clean_and_exit 1 "ERROR: Database ${DB_SPEC} does not exist." + fi + fi + + echo "Restoring Database $DB_SPEC And Grants" + restore_single_db $DB_SPEC $TMP_DIR + if [[ "$?" -eq 0 ]]; then + echo "Single database restored successfully." + else + clean_and_exit 1 "ERROR: Single database restore failed." + fi + clean_and_exit 0 "" + else + echo "Restoring All The Databases. This could take a few minutes..." + restore_all_dbs $TMP_DIR + if [[ "$?" -eq 0 ]]; then + echo "All databases restored successfully." + else + clean_and_exit 1 "ERROR: Database restore failed." + fi + clean_and_exit 0 "" + fi + ;; + "delete_archive") + if [[ ${#ARGS[@]} -lt 2 || ${#ARGS[@]} -gt 3 ]]; then + usage 1 + elif [[ ${#ARGS[@]} -eq 2 ]]; then + delete_archive ${ARGS[1]} + else + delete_archive ${ARGS[1]} ${ARGS[2]} + fi + ;; + *) + usage 1 + ;; + esac + + clean_and_exit 0 "" +} +{{- end }} diff --git a/helm-toolkit/templates/snippets/_custom_job_annotations.tpl b/helm-toolkit/templates/snippets/_custom_job_annotations.tpl new file mode 100644 index 0000000000..fc426142fd --- /dev/null +++ b/helm-toolkit/templates/snippets/_custom_job_annotations.tpl @@ -0,0 +1,76 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Adds custom annotations to the job spec of a component. +examples: + - values: | + annotations: + job: + default: + custom.tld/key: "value" + custom.tld/key2: "value2" + keystone_domain_manage: + another.tld/foo: "bar" + usage: | + {{ tuple "keystone_domain_manage" . | include "helm-toolkit.snippets.custom_job_annotations" }} + return: | + another.tld/foo: bar + - values: | + annotations: + job: + default: + custom.tld/key: "value" + custom.tld/key2: "value2" + keystone_domain_manage: + another.tld/foo: "bar" + usage: | + {{ tuple "keystone_bootstrap" . | include "helm-toolkit.snippets.custom_job_annotations" }} + return: | + custom.tld/key: "value" + custom.tld/key2: "value2" + - values: | + annotations: + job: + default: + custom.tld/key: "value" + custom.tld/key2: "value2" + keystone_domain_manage: + another.tld/foo: "bar" + keystone_bootstrap: + usage: | + {{ tuple "keystone_bootstrap" . | include "helm-toolkit.snippets.custom_job_annotations" }} + return: | + custom.tld/key: "value" + custom.tld/key2: "value2" +*/}} + +{{- define "helm-toolkit.snippets.custom_job_annotations" -}} +{{- $envAll := index . 1 -}} +{{- $component := index . 0 | replace "-" "_" -}} +{{- if (hasKey $envAll.Values "annotations") -}} +{{- if (hasKey $envAll.Values.annotations "job") -}} +{{- $annotationsMap := $envAll.Values.annotations.job -}} +{{- $defaultAnnotations := dict -}} +{{- if (hasKey $annotationsMap "default" ) -}} +{{- $defaultAnnotations = $annotationsMap.default -}} +{{- end -}} +{{- $annotations := index $annotationsMap $component | default $defaultAnnotations -}} +{{- if (not (empty $annotations)) -}} +{{- toYaml $annotations -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl b/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl new file mode 100644 index 0000000000..ecff6e96a6 --- /dev/null +++ b/helm-toolkit/templates/snippets/_custom_pod_annotations.tpl @@ -0,0 +1,76 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Adds custom annotations to the pod spec of a component. +examples: + - values: | + annotations: + pod: + default: + custom.tld/key: "value" + custom.tld/key2: "value2" + nova_compute: + another.tld/foo: "bar" + usage: | + {{ tuple "nova_compute" . | include "helm-toolkit.snippets.custom_pod_annotations" }} + return: | + another.tld/foo: bar + - values: | + annotations: + pod: + default: + custom.tld/key: "value" + custom.tld/key2: "value2" + nova_compute: + another.tld/foo: "bar" + usage: | + {{ tuple "nova_api" . | include "helm-toolkit.snippets.custom_pod_annotations" }} + return: | + custom.tld/key: "value" + custom.tld/key2: "value2" + - values: | + annotations: + pod: + default: + custom.tld/key: "value" + custom.tld/key2: "value2" + nova_compute: + another.tld/foo: "bar" + nova_api: + usage: | + {{ tuple "nova_api" . | include "helm-toolkit.snippets.custom_pod_annotations" }} + return: | + custom.tld/key: "value" + custom.tld/key2: "value2" +*/}} + +{{- define "helm-toolkit.snippets.custom_pod_annotations" -}} +{{- $component := index . 0 -}} +{{- $envAll := index . 1 -}} +{{- if (hasKey $envAll.Values "annotations") -}} +{{- if (hasKey $envAll.Values.annotations "pod") -}} +{{- $annotationsMap := $envAll.Values.annotations.pod -}} +{{- $defaultAnnotations := dict -}} +{{- if (hasKey $annotationsMap "default" ) -}} +{{- $defaultAnnotations = $annotationsMap.default -}} +{{- end -}} +{{- $annotations := index $annotationsMap $component | default $defaultAnnotations -}} +{{- if (not (empty $annotations)) -}} +{{- toYaml $annotations -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl b/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl new file mode 100644 index 0000000000..19c438088b --- /dev/null +++ b/helm-toolkit/templates/snippets/_custom_secret_annotations.tpl @@ -0,0 +1,81 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Adds custom annotations to the secret spec of a component. +examples: + - values: | + annotations: + secret: + default: + custom.tld/key: "value" + custom.tld/key2: "value2" + identity: + admin: + another.tld/foo: "bar" + usage: | + {{ tuple "identity" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }} + return: | + another.tld/foo: bar + - values: | + annotations: + secret: + default: + custom.tld/key: "value" + custom.tld/key2: "value2" + identity: + admin: + another.tld/foo: "bar" + usage: | + {{ tuple "oslo_db" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }} + return: | + custom.tld/key: "value" + custom.tld/key2: "value2" + - values: | + annotations: + secret: + default: + custom.tld/key: "value" + custom.tld/key2: "value2" + identity: + admin: + another.tld/foo: "bar" + oslo_db: + admin: + usage: | + {{ tuple "oslo_db" "admin" . | include "helm-toolkit.snippets.custom_secret_annotations" }} + return: | + custom.tld/key: "value" + custom.tld/key2: "value2" +*/}} + +{{- define "helm-toolkit.snippets.custom_secret_annotations" -}} +{{- $secretType := index . 0 -}} +{{- $userClass := index . 1 | replace "-" "_" -}} +{{- $envAll := index . 2 -}} +{{- if (hasKey $envAll.Values "annotations") -}} +{{- if (hasKey $envAll.Values.annotations "secret") -}} +{{- $annotationsMap := index $envAll.Values.annotations.secret $secretType | default dict -}} +{{- $defaultAnnotations := dict -}} +{{- if (hasKey $envAll.Values.annotations.secret "default" ) -}} +{{- $defaultAnnotations = $envAll.Values.annotations.secret.default -}} +{{- end -}} +{{- $annotations := index $annotationsMap $userClass | default $defaultAnnotations -}} +{{- if (not (empty $annotations)) -}} +{{- toYaml $annotations -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/helm-toolkit/templates/snippets/_image.tpl b/helm-toolkit/templates/snippets/_image.tpl new file mode 100644 index 0000000000..678b8447f8 --- /dev/null +++ b/helm-toolkit/templates/snippets/_image.tpl @@ -0,0 +1,60 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Resolves an image reference to a string, and its pull policy +values: | + images: + tags: + test_image: docker.io/port/test:version-foo + image_foo: quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal + pull_policy: IfNotPresent + local_registry: + active: true + exclude: + - image_foo + endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 +usage: | + {{ tuple . "test_image" | include "helm-toolkit.snippets.image" }} +return: | + image: "localhost:5000/docker.io/port/test:version-foo" + imagePullPolicy: IfNotPresent +*/}} + +{{- define "helm-toolkit.snippets.image" -}} +{{- $envAll := index . 0 -}} +{{- $image := index . 1 -}} +{{- $imageTag := index $envAll.Values.images.tags $image -}} +{{- if and ($envAll.Values.images.local_registry.active) (not (has $image $envAll.Values.images.local_registry.exclude )) -}} +{{- $registryPrefix := printf "%s:%s" (tuple "local_image_registry" "node" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup") (tuple "local_image_registry" "node" "registry" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup") -}} +image: {{ printf "%s/%s" $registryPrefix $imageTag | quote }} +{{- else -}} +image: {{ $imageTag | quote }} +{{- end }} +imagePullPolicy: {{ $envAll.Values.images.pull_policy }} +{{- end -}} diff --git a/helm-toolkit/templates/snippets/_keystone_openrc_env_vars.tpl b/helm-toolkit/templates/snippets/_keystone_openrc_env_vars.tpl new file mode 100644 index 0000000000..2f209fe63d --- /dev/null +++ b/helm-toolkit/templates/snippets/_keystone_openrc_env_vars.tpl @@ -0,0 +1,142 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Returns a set of container enviorment variables, equivlant to an openrc for + use with keystone based command line clients. +values: | + secrets: + identity: + admin: example-keystone-admin +usage: | + {{ include "helm-toolkit.snippets.keystone_openrc_env_vars" ( dict "ksUserSecret" .Values.secrets.identity.admin ) }} +return: | + - name: OS_IDENTITY_API_VERSION + value: "3" + - name: OS_AUTH_URL + valueFrom: + secretKeyRef: + name: example-keystone-admin + key: OS_AUTH_URL + - name: OS_REGION_NAME + valueFrom: + secretKeyRef: + name: example-keystone-admin + key: OS_REGION_NAME + - name: OS_INTERFACE + valueFrom: + secretKeyRef: + name: example-keystone-admin + key: OS_INTERFACE + - name: OS_ENDPOINT_TYPE + valueFrom: + secretKeyRef: + name: example-keystone-admin + key: OS_INTERFACE + - name: OS_PROJECT_DOMAIN_NAME + valueFrom: + secretKeyRef: + name: example-keystone-admin + key: OS_PROJECT_DOMAIN_NAME + - name: OS_PROJECT_NAME + valueFrom: + secretKeyRef: + name: example-keystone-admin + key: OS_PROJECT_NAME + - name: OS_USER_DOMAIN_NAME + valueFrom: + secretKeyRef: + name: example-keystone-admin + key: OS_USER_DOMAIN_NAME + - name: OS_USERNAME + valueFrom: + secretKeyRef: + name: example-keystone-admin + key: OS_USERNAME + - name: OS_PASSWORD + valueFrom: + secretKeyRef: + name: example-keystone-admin + key: OS_PASSWORD + - name: OS_CACERT + valueFrom: + secretKeyRef: + name: example-keystone-admin + key: OS_CACERT +*/}} + +{{- define "helm-toolkit.snippets.keystone_openrc_env_vars" }} +{{- $useCA := .useCA -}} +{{- $ksUserSecret := .ksUserSecret }} +- name: OS_IDENTITY_API_VERSION + value: "3" +- name: OS_AUTH_URL + valueFrom: + secretKeyRef: + name: {{ $ksUserSecret }} + key: OS_AUTH_URL +- name: OS_REGION_NAME + valueFrom: + secretKeyRef: + name: {{ $ksUserSecret }} + key: OS_REGION_NAME +- name: OS_INTERFACE + valueFrom: + secretKeyRef: + name: {{ $ksUserSecret }} + key: OS_INTERFACE +- name: OS_ENDPOINT_TYPE + valueFrom: + secretKeyRef: + name: {{ $ksUserSecret }} + key: OS_INTERFACE +- name: OS_PROJECT_DOMAIN_NAME + valueFrom: + secretKeyRef: + name: {{ $ksUserSecret }} + key: OS_PROJECT_DOMAIN_NAME +- name: OS_PROJECT_NAME + valueFrom: + secretKeyRef: + name: {{ $ksUserSecret }} + key: OS_PROJECT_NAME +- name: OS_USER_DOMAIN_NAME + valueFrom: + secretKeyRef: + name: {{ $ksUserSecret }} + key: OS_USER_DOMAIN_NAME +- name: OS_USERNAME + valueFrom: + secretKeyRef: + name: {{ $ksUserSecret }} + key: OS_USERNAME +- name: OS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $ksUserSecret }} + key: OS_PASSWORD +- name: OS_DEFAULT_DOMAIN + valueFrom: + secretKeyRef: + name: {{ $ksUserSecret }} + key: OS_DEFAULT_DOMAIN +{{- if $useCA }} +- name: OS_CACERT + valueFrom: + secretKeyRef: + name: {{ $ksUserSecret }} + key: OS_CACERT +{{- end }} +{{- end }} diff --git a/helm-toolkit/templates/snippets/_keystone_secret_openrc.tpl b/helm-toolkit/templates/snippets/_keystone_secret_openrc.tpl new file mode 100644 index 0000000000..f6276576c8 --- /dev/null +++ b/helm-toolkit/templates/snippets/_keystone_secret_openrc.tpl @@ -0,0 +1,32 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "helm-toolkit.snippets.keystone_secret_openrc" }} +{{- $userClass := index . 0 -}} +{{- $identityEndpoint := index . 1 -}} +{{- $context := index . 2 -}} +{{- $userContext := index $context.Values.endpoints.identity.auth $userClass }} +OS_AUTH_URL: {{ tuple "identity" $identityEndpoint "api" $context | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | b64enc }} +OS_REGION_NAME: {{ $userContext.region_name | b64enc }} +OS_INTERFACE: {{ $userContext.interface | default "internal" | b64enc }} +OS_PROJECT_DOMAIN_NAME: {{ $userContext.project_domain_name | b64enc }} +OS_PROJECT_NAME: {{ $userContext.project_name | b64enc }} +OS_USER_DOMAIN_NAME: {{ $userContext.user_domain_name | b64enc }} +OS_USERNAME: {{ $userContext.username | b64enc }} +OS_PASSWORD: {{ $userContext.password | b64enc }} +OS_DEFAULT_DOMAIN: {{ $userContext.default_domain_id | default "default" | b64enc }} +{{- if $userContext.cacert }} +OS_CACERT: {{ $userContext.cacert | b64enc }} +{{- end }} +{{- end }} diff --git a/helm-toolkit/templates/snippets/_keystone_user_create_env_vars.tpl b/helm-toolkit/templates/snippets/_keystone_user_create_env_vars.tpl new file mode 100644 index 0000000000..648711beb2 --- /dev/null +++ b/helm-toolkit/templates/snippets/_keystone_user_create_env_vars.tpl @@ -0,0 +1,90 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Returns a set of container enviorment variables, for use with the keystone + user management jobs. +values: | + secrets: + identity: + service_user: example-keystone-user +usage: | + {{ include "helm-toolkit.snippets.keystone_user_create_env_vars" ( dict "ksUserSecret" .Values.secrets.identity.service_user "useCA" true ) }} +return: | + - name: SERVICE_OS_REGION_NAME + valueFrom: + secretKeyRef: + name: example-keystone-user + key: OS_REGION_NAME + - name: SERVICE_OS_PROJECT_DOMAIN_NAME + valueFrom: + secretKeyRef: + name: example-keystone-user + key: OS_PROJECT_DOMAIN_NAME + - name: SERVICE_OS_PROJECT_NAME + valueFrom: + secretKeyRef: + name: example-keystone-user + key: OS_PROJECT_NAME + - name: SERVICE_OS_USER_DOMAIN_NAME + valueFrom: + secretKeyRef: + name: example-keystone-user + key: OS_USER_DOMAIN_NAME + - name: SERVICE_OS_USERNAME + valueFrom: + secretKeyRef: + name: example-keystone-user + key: OS_USERNAME + - name: SERVICE_OS_PASSWORD + valueFrom: + secretKeyRef: + name: example-keystone-user + key: OS_PASSWORD +*/}} + +{{- define "helm-toolkit.snippets.keystone_user_create_env_vars" }} +{{- $ksUserSecret := .ksUserSecret }} +- name: SERVICE_OS_REGION_NAME + valueFrom: + secretKeyRef: + name: {{ $ksUserSecret }} + key: OS_REGION_NAME +- name: SERVICE_OS_PROJECT_DOMAIN_NAME + valueFrom: + secretKeyRef: + name: {{ $ksUserSecret }} + key: OS_PROJECT_DOMAIN_NAME +- name: SERVICE_OS_PROJECT_NAME + valueFrom: + secretKeyRef: + name: {{ $ksUserSecret }} + key: OS_PROJECT_NAME +- name: SERVICE_OS_USER_DOMAIN_NAME + valueFrom: + secretKeyRef: + name: {{ $ksUserSecret }} + key: OS_USER_DOMAIN_NAME +- name: SERVICE_OS_USERNAME + valueFrom: + secretKeyRef: + name: {{ $ksUserSecret }} + key: OS_USERNAME +- name: SERVICE_OS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $ksUserSecret }} + key: OS_PASSWORD +{{- end }} diff --git a/helm-toolkit/templates/snippets/_kubernetes_apparmor_configmap.tpl b/helm-toolkit/templates/snippets/_kubernetes_apparmor_configmap.tpl new file mode 100644 index 0000000000..8ca102806d --- /dev/null +++ b/helm-toolkit/templates/snippets/_kubernetes_apparmor_configmap.tpl @@ -0,0 +1,68 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Renders a configmap used for loading custom AppArmor profiles. +values: | + pod: + mandatory_access_control: + type: apparmor + configmap_apparmor: true + apparmor_profiles: |- + my_apparmor-v1.profile: |- + #include + profile my-apparmor-v1 flags=(attach_disconnected,mediate_deleted) { + + } +usage: | + {{ dict "envAll" . "component" "myComponent" | include "helm-toolkit.snippets.kubernetes_apparmor_configmap" }} +return: | +apiVersion: v1 +kind: ConfigMap +metadata: + name: releaseName-myComponent-apparmor + namespace: myNamespace +data: + my_apparmor-v1.profile: |- + #include + profile my-apparmor-v1 flags=(attach_disconnected,mediate_deleted) { + + } +*/}} +{{- define "helm-toolkit.snippets.kubernetes_apparmor_configmap" -}} +{{- $envAll := index . "envAll" -}} +{{- $component := index . "component" -}} +{{- if hasKey $envAll.Values.pod "mandatory_access_control" -}} +{{- if hasKey $envAll.Values.pod.mandatory_access_control "type" -}} +{{- if eq $envAll.Values.pod.mandatory_access_control.type "apparmor" -}} +{{- if hasKey $envAll.Values.pod.mandatory_access_control "configmap_apparmor" -}} +{{- if $envAll.Values.pod.mandatory_access_control.configmap_apparmor }} +{{- $mapName := printf "%s-%s-%s" $envAll.Release.Name $component "apparmor" -}} +{{- if $envAll.Values.conf.apparmor_profiles }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $mapName }} + namespace: {{ $envAll.Release.Namespace }} +data: +{{ $envAll.Values.conf.apparmor_profiles | toYaml | indent 2 }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/helm-toolkit/templates/snippets/_kubernetes_apparmor_loader_init_container.tpl b/helm-toolkit/templates/snippets/_kubernetes_apparmor_loader_init_container.tpl new file mode 100644 index 0000000000..f231fe6598 --- /dev/null +++ b/helm-toolkit/templates/snippets/_kubernetes_apparmor_loader_init_container.tpl @@ -0,0 +1,75 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Renders the init container used for apparmor loading. +values: | + images: + tags: + apparmor_loader: my-repo.io/apparmor-loader:1.0.0 + pod: + mandatory_access_control: + type: apparmor + configmap_apparmor: true + apparmor-loader: unconfined +usage: | + {{ dict "envAll" . | include "helm-toolkit.snippets.kubernetes_apparmor_loader_init_container" }} +return: | + - name: apparmor-loader + image: my-repo.io/apparmor-loader:1.0.0 + args: + - /profiles + securityContext: + privileged: true + volumeMounts: + - name: sys + mountPath: /sys + readOnly: true + - name: includes + mountPath: /etc/apparmor.d + readOnly: true + - name: profiles + mountPath: /profiles + readOnly: true +*/}} +{{- define "helm-toolkit.snippets.kubernetes_apparmor_loader_init_container" -}} +{{- $envAll := index . "envAll" -}} +{{- if hasKey $envAll.Values.pod "mandatory_access_control" -}} +{{- if hasKey $envAll.Values.pod.mandatory_access_control "type" -}} +{{- if hasKey $envAll.Values.pod.mandatory_access_control "configmap_apparmor" -}} +{{- if eq $envAll.Values.pod.mandatory_access_control.type "apparmor" -}} +{{- if $envAll.Values.pod.mandatory_access_control.configmap_apparmor }} +- name: apparmor-loader + image: {{ $envAll.Values.images.tags.apparmor_loader }} + args: + - /profiles + securityContext: + privileged: true + volumeMounts: + - name: sys + mountPath: /sys + readOnly: true + - name: includes + mountPath: /etc/apparmor.d + readOnly: true + - name: profiles + mountPath: /profiles + readOnly: true +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/helm-toolkit/templates/snippets/_kubernetes_apparmor_volumes.tpl b/helm-toolkit/templates/snippets/_kubernetes_apparmor_volumes.tpl new file mode 100644 index 0000000000..baebaa3cba --- /dev/null +++ b/helm-toolkit/templates/snippets/_kubernetes_apparmor_volumes.tpl @@ -0,0 +1,68 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Renders the volumes used by the apparmor loader. +values: | + pod: + mandatory_access_control: + type: apparmor + configmap_apparmor: true +inputs: | + envAll: "Environment or Context." + component: "Name of the component used for the name of configMap." + requireSys: "Boolean. True if it needs the hostpath /sys in volumes." +usage: | + {{ dict "envAll" . "component" "keystone" "requireSys" true | include "helm-toolkit.snippets.kubernetes_apparmor_volumes" }} +return: | +- name: sys + hostPath: + path: /sys +- name: includes + hostPath: + path: /etc/apparmor.d +- name: profiles + configMap: + name: RELEASENAME-keystone-apparmor + defaultMode: 0555 +*/}} +{{- define "helm-toolkit.snippets.kubernetes_apparmor_volumes" -}} +{{- $envAll := index . "envAll" -}} +{{- $component := index . "component" -}} +{{- $requireSys := index . "requireSys" | default false -}} +{{- $configName := printf "%s-%s-%s" $envAll.Release.Name $component "apparmor" -}} +{{- if hasKey $envAll.Values.pod "mandatory_access_control" -}} +{{- if hasKey $envAll.Values.pod.mandatory_access_control "type" -}} +{{- if hasKey $envAll.Values.pod.mandatory_access_control "configmap_apparmor" -}} +{{- if eq $envAll.Values.pod.mandatory_access_control.type "apparmor" -}} +{{- if $envAll.Values.pod.mandatory_access_control.configmap_apparmor }} +{{- if $requireSys }} +- name: sys + hostPath: + path: /sys +{{- end }} +- name: includes + hostPath: + path: /etc/apparmor.d +- name: profiles + configMap: + name: {{ $configName | quote }} + defaultMode: 0555 +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/helm-toolkit/templates/snippets/_kubernetes_container_security_context.tpl b/helm-toolkit/templates/snippets/_kubernetes_container_security_context.tpl new file mode 100644 index 0000000000..4741497e2b --- /dev/null +++ b/helm-toolkit/templates/snippets/_kubernetes_container_security_context.tpl @@ -0,0 +1,48 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Renders securityContext for a Kubernetes container. + For container level, see here: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.13/#securitycontext-v1-core +examples: + - values: | + pod: + security_context: + myApp: + container: + foo: + runAsUser: 34356 + readOnlyRootFilesystem: true + usage: | + {{ dict "envAll" . "application" "myApp" "container" "foo" | include "helm-toolkit.snippets.kubernetes_container_security_context" }} + return: | + securityContext: + readOnlyRootFilesystem: true + runAsUser: 34356 +*/}} + +{{- define "helm-toolkit.snippets.kubernetes_container_security_context" -}} +{{- $envAll := index . "envAll" -}} +{{- $application := index . "application" -}} +{{- $container := index . "container" -}} +{{- if hasKey $envAll.Values.pod "security_context" }} +{{- if hasKey ( index $envAll.Values.pod.security_context ) $application }} +{{- if hasKey ( index $envAll.Values.pod.security_context $application "container" ) $container }} +securityContext: +{{ toYaml ( index $envAll.Values.pod.security_context $application "container" $container ) | indent 2 }} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/helm-toolkit/templates/snippets/_kubernetes_entrypoint_init_container.tpl b/helm-toolkit/templates/snippets/_kubernetes_entrypoint_init_container.tpl new file mode 100644 index 0000000000..ad628daca1 --- /dev/null +++ b/helm-toolkit/templates/snippets/_kubernetes_entrypoint_init_container.tpl @@ -0,0 +1,209 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Returns a container definition for use with the kubernetes-entrypoint image + from stackanetes. +values: | + images: + tags: + dep_check: quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal + pull_policy: IfNotPresent + local_registry: + active: true + exclude: + - dep_check + dependencies: + dynamic: + common: + local_image_registry: + jobs: + - calico-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + calico_node: + services: + - endpoint: internal + service: etcd + custom_resources: + - apiVersion: argoproj.io/v1alpha1 + kind: Workflow + name: wf-example + fields: + - key: "status.phase" + value: "Succeeded" + endpoints: + local_image_registry: + namespace: docker-registry + hosts: + default: localhost + node: localhost + etcd: + hosts: + default: etcd + # NOTE (portdirect): if the stanza, or a portion of it, under `pod` is not + # specififed then the following will be used as defaults: + # pod: + # security_context: + # kubernetes_entrypoint: + # container: + # kubernetes_entrypoint: + # runAsUser: 65534 + # readOnlyRootFilesystem: true + # allowPrivilegeEscalation: false + pod: + security_context: + kubernetes_entrypoint: + container: + kubernetes_entrypoint: + runAsUser: 0 + readOnlyRootFilesystem: false +usage: | + {{ tuple . "calico_node" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" }} +return: | + - name: init + image: "quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal" + imagePullPolicy: IfNotPresent + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false + runAsUser: 0 + + env: + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: INTERFACE_NAME + value: eth0 + - name: PATH + value: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/ + - name: DEPENDENCY_SERVICE + value: "default:etcd,docker-registry:localhost" + - name: DEPENDENCY_JOBS + value: "calico-image-repo-sync" + - name: DEPENDENCY_DAEMONSET + value: "" + - name: DEPENDENCY_CONTAINER + value: "" + - name: DEPENDENCY_POD_JSON + value: "" + - name: DEPENDENCY_CUSTOM_RESOURCE + value: "[{\"apiVersion\":\"argoproj.io/v1alpha1\",\"kind\":\"Workflow\",\"namespace\":\"default\",\"name\":\"wf-example\",\"fields\":[{\"key\":\"status.phase\",\"value\":\"Succeeded\"}]}]" + command: + - kubernetes-entrypoint + volumeMounts: + [] +*/}} + +{{- define "helm-toolkit.snippets.kubernetes_entrypoint_init_container._default_security_context" -}} +Values: + pod: + security_context: + kubernetes_entrypoint: + container: + kubernetes_entrypoint: + runAsUser: 65534 + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false +{{- end -}} + +{{- define "helm-toolkit.snippets.kubernetes_entrypoint_init_container" -}} +{{- $envAll := index . 0 -}} +{{- $component := index . 1 -}} +{{- $mounts := index . 2 -}} + +{{- $_ := set $envAll.Values "__kubernetes_entrypoint_init_container" dict -}} +{{- $_ := set $envAll.Values.__kubernetes_entrypoint_init_container "deps" dict -}} +{{- if and ($envAll.Values.images.local_registry.active) (ne $component "image_repo_sync") -}} +{{- if eq $component "pod_dependency" -}} +{{- $_ := include "helm-toolkit.utils.merge" ( tuple $envAll.Values.__kubernetes_entrypoint_init_container.deps ( index $envAll.Values.pod_dependency ) $envAll.Values.dependencies.dynamic.common.local_image_registry ) -}} +{{- else -}} +{{- $_ := include "helm-toolkit.utils.merge" ( tuple $envAll.Values.__kubernetes_entrypoint_init_container.deps ( index $envAll.Values.dependencies.static $component ) $envAll.Values.dependencies.dynamic.common.local_image_registry ) -}} +{{- end -}} +{{- else -}} +{{- if eq $component "pod_dependency" -}} +{{- $_ := set $envAll.Values.__kubernetes_entrypoint_init_container "deps" ( index $envAll.Values.pod_dependency ) -}} +{{- else -}} +{{- $_ := set $envAll.Values.__kubernetes_entrypoint_init_container "deps" ( index $envAll.Values.dependencies.static $component ) -}} +{{- end -}} +{{- end -}} + +{{- if and ($envAll.Values.manifests.job_rabbit_init) (hasKey $envAll.Values.dependencies "dynamic") -}} +{{- if $envAll.Values.dependencies.dynamic.job_rabbit_init -}} +{{- if eq $component "pod_dependency" -}} +{{- $_ := include "helm-toolkit.utils.merge" ( tuple $envAll.Values.__kubernetes_entrypoint_init_container.deps ( index $envAll.Values.pod_dependency ) (index $envAll.Values.dependencies.dynamic.job_rabbit_init $component) ) -}} +{{- else -}} +{{- $_ := include "helm-toolkit.utils.merge" ( tuple $envAll.Values.__kubernetes_entrypoint_init_container.deps ( index $envAll.Values.dependencies.static $component ) (index $envAll.Values.dependencies.dynamic.job_rabbit_init $component)) -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{- $deps := $envAll.Values.__kubernetes_entrypoint_init_container.deps }} +{{- range $deps.custom_resources }} +{{- $_ := set . "namespace" $envAll.Release.Namespace -}} +{{- end -}} +{{- $default_security_context := include "helm-toolkit.snippets.kubernetes_entrypoint_init_container._default_security_context" . | fromYaml }} +{{- $patchedEnvAll := mergeOverwrite $default_security_context $envAll }} +- name: init +{{ tuple $envAll "dep_check" | include "helm-toolkit.snippets.image" | indent 2 }} +{{- dict "envAll" $patchedEnvAll "application" "kubernetes_entrypoint" "container" "kubernetes_entrypoint" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 2 }} + env: + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + - name: INTERFACE_NAME + value: eth0 + - name: PATH + value: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/ + - name: DEPENDENCY_SERVICE + value: "{{ tuple $deps.services $envAll | include "helm-toolkit.utils.comma_joined_service_list" }}" +{{- if $deps.jobs -}} + {{- if kindIs "string" (index $deps.jobs 0) }} + - name: DEPENDENCY_JOBS + value: "{{ include "helm-toolkit.utils.joinListWithComma" $deps.jobs }}" + {{- else }} + - name: DEPENDENCY_JOBS_JSON + value: {{- toJson $deps.jobs | quote -}} + {{- end -}} +{{- end }} + - name: DEPENDENCY_DAEMONSET + value: "{{ include "helm-toolkit.utils.joinListWithComma" $deps.daemonset }}" + - name: DEPENDENCY_CONTAINER + value: "{{ include "helm-toolkit.utils.joinListWithComma" $deps.container }}" + - name: DEPENDENCY_POD_JSON + value: {{ if $deps.pod }}{{ toJson $deps.pod | quote }}{{ else }}""{{ end }} + - name: DEPENDENCY_CUSTOM_RESOURCE + value: {{ if $deps.custom_resources }}{{ toJson $deps.custom_resources | quote }}{{ else }}""{{ end }} + command: + - kubernetes-entrypoint + volumeMounts: +{{ toYaml $mounts | indent 4 }} +{{- end -}} diff --git a/helm-toolkit/templates/snippets/_kubernetes_kubectl_params.tpl b/helm-toolkit/templates/snippets/_kubernetes_kubectl_params.tpl new file mode 100644 index 0000000000..34a7da33a4 --- /dev/null +++ b/helm-toolkit/templates/snippets/_kubernetes_kubectl_params.tpl @@ -0,0 +1,20 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "helm-toolkit.snippets.kubernetes_kubectl_params" -}} +{{- $envAll := index . 0 -}} +{{- $application := index . 1 -}} +{{- $component := index . 2 -}} +{{ print "-l application=" $application " -l component=" $component }} +{{- end -}} diff --git a/helm-toolkit/templates/snippets/_kubernetes_mandatory_access_control_annotation.tpl b/helm-toolkit/templates/snippets/_kubernetes_mandatory_access_control_annotation.tpl new file mode 100644 index 0000000000..92d3ea5cbf --- /dev/null +++ b/helm-toolkit/templates/snippets/_kubernetes_mandatory_access_control_annotation.tpl @@ -0,0 +1,60 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Renders mandatory access control annotations for a list of containers + driven by values.yaml. As of now, it can only generate an apparmor + annotation, but in the future could generate others. +values: | + pod: + mandatory_access_control: + type: apparmor + myPodName: + myContainerName: localhost/myAppArmor + mySecondContainerName: localhost/secondProfile # optional + myThirdContainerName: localhost/thirdProfile # optional +usage: | + {{ dict "envAll" . "podName" "myPodName" "containerNames" (list "myContainerName" "mySecondContainerName" "myThirdContainerName") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" }} +return: | + container.apparmor.security.beta.kubernetes.io/myContainerName: localhost/myAppArmor + container.apparmor.security.beta.kubernetes.io/mySecondContainerName: localhost/secondProfile + container.apparmor.security.beta.kubernetes.io/myThirdContainerName: localhost/thirdProfile +note: | + The number of container underneath is a variable arguments. It loops through + all the container names specified. +*/}} +{{- define "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" -}} +{{- $envAll := index . "envAll" -}} +{{- $podName := index . "podName" -}} +{{- $containerNames := index . "containerNames" -}} +{{- if hasKey $envAll.Values.pod "mandatory_access_control" -}} +{{- if hasKey $envAll.Values.pod.mandatory_access_control "type" -}} +{{- $macType := $envAll.Values.pod.mandatory_access_control.type -}} +{{- if $macType -}} +{{- if eq $macType "apparmor" -}} +{{- if hasKey $envAll.Values.pod.mandatory_access_control $podName -}} +{{- range $name := $containerNames -}} +{{- $apparmorProfile := index $envAll.Values.pod.mandatory_access_control $podName $name -}} +{{- if $apparmorProfile }} +container.apparmor.security.beta.kubernetes.io/{{ $name }}: {{ $apparmorProfile }} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} + diff --git a/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl b/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl new file mode 100644 index 0000000000..5c2dedb06f --- /dev/null +++ b/helm-toolkit/templates/snippets/_kubernetes_metadata_labels.tpl @@ -0,0 +1,54 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Renders a set of standardised labels +values: | + release_group: null + pod: + labels: + default: + label1.example.com: value + bar: + label2.example.com: bar +usage: | + {{ tuple . "foo" "bar" | include "helm-toolkit.snippets.kubernetes_metadata_labels" }} +return: | + release_group: RELEASE-NAME + application: foo + component: bar + label1.example.com: value + label2.example.com: bar +*/}} + +{{- define "helm-toolkit.snippets.kubernetes_metadata_labels" -}} +{{- $envAll := index . 0 -}} +{{- $application := index . 1 -}} +{{- $component := index . 2 -}} +release_group: {{ $envAll.Values.release_group | default $envAll.Release.Name }} +application: {{ $application }} +component: {{ $component }} +app.kubernetes.io/name: {{ $application }} +app.kubernetes.io/component: {{ $component }} +app.kubernetes.io/instance: {{ $envAll.Values.release_group | default $envAll.Release.Name }} +{{- if ($envAll.Values.pod).labels }} +{{- if hasKey $envAll.Values.pod.labels $component }} +{{ index $envAll.Values.pod "labels" $component | toYaml }} +{{- end -}} +{{- if hasKey $envAll.Values.pod.labels "default" }} +{{ $envAll.Values.pod.labels.default | toYaml }} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/helm-toolkit/templates/snippets/_kubernetes_pod_anti_affinity.tpl b/helm-toolkit/templates/snippets/_kubernetes_pod_anti_affinity.tpl new file mode 100644 index 0000000000..fabbcf8d99 --- /dev/null +++ b/helm-toolkit/templates/snippets/_kubernetes_pod_anti_affinity.tpl @@ -0,0 +1,89 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Renders kubernetes anti affinity rules, this function supports both hard + 'requiredDuringSchedulingIgnoredDuringExecution' and soft + 'preferredDuringSchedulingIgnoredDuringExecution' types. +values: | + pod: + affinity: + anti: + topologyKey: + default: kubernetes.io/hostname + type: + default: requiredDuringSchedulingIgnoredDuringExecution + weight: + default: 10 +usage: | + {{ tuple . "appliction_x" "component_y" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" }} +return: | + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: release_group + operator: In + values: + - RELEASE-NAME + - key: application + operator: In + values: + - appliction_x + - key: component + operator: In + values: + - component_y + topologyKey: kubernetes.io/hostname +*/}} + +{{- define "helm-toolkit.snippets.kubernetes_pod_anti_affinity._match_expressions" -}} +{{- $envAll := index . "envAll" -}} +{{- $application := index . "application" -}} +{{- $component := index . "component" -}} +{{- $expressionRelease := dict "key" "release_group" "operator" "In" "values" ( list ( $envAll.Values.release_group | default $envAll.Release.Name ) ) -}} +{{- $expressionApplication := dict "key" "application" "operator" "In" "values" ( list $application ) -}} +{{- $expressionComponent := dict "key" "component" "operator" "In" "values" ( list $component ) -}} +{{- list $expressionRelease $expressionApplication $expressionComponent | toYaml }} +{{- end -}} + +{{- define "helm-toolkit.snippets.kubernetes_pod_anti_affinity" -}} +{{- $envAll := index . 0 -}} +{{- $application := index . 1 -}} +{{- $component := index . 2 -}} +{{- $antiAffinityType := index $envAll.Values.pod.affinity.anti.type $component | default $envAll.Values.pod.affinity.anti.type.default }} +{{- $antiAffinityKey := index $envAll.Values.pod.affinity.anti.topologyKey $component | default $envAll.Values.pod.affinity.anti.topologyKey.default }} +podAntiAffinity: +{{- $matchExpressions := include "helm-toolkit.snippets.kubernetes_pod_anti_affinity._match_expressions" ( dict "envAll" $envAll "application" $application "component" $component ) -}} +{{- if eq $antiAffinityType "preferredDuringSchedulingIgnoredDuringExecution" }} + {{ $antiAffinityType }}: + - podAffinityTerm: + labelSelector: + matchExpressions: +{{ $matchExpressions | indent 10 }} + topologyKey: {{ $antiAffinityKey }} +{{- if $envAll.Values.pod.affinity.anti.weight }} + weight: {{ index $envAll.Values.pod.affinity.anti.weight $component | default $envAll.Values.pod.affinity.anti.weight.default }} +{{- else }} + weight: 10 +{{- end -}} +{{- else if eq $antiAffinityType "requiredDuringSchedulingIgnoredDuringExecution" }} + {{ $antiAffinityType }}: + - labelSelector: + matchExpressions: +{{ $matchExpressions | indent 8 }} + topologyKey: {{ $antiAffinityKey }} +{{- end -}} +{{- end -}} diff --git a/helm-toolkit/templates/snippets/_kubernetes_pod_image_pull_secret.tpl b/helm-toolkit/templates/snippets/_kubernetes_pod_image_pull_secret.tpl new file mode 100644 index 0000000000..74173dcef4 --- /dev/null +++ b/helm-toolkit/templates/snippets/_kubernetes_pod_image_pull_secret.tpl @@ -0,0 +1,45 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Renders image pull secrets for a pod +values: | + pod: + image_pull_secrets: + default: + - name: some-pull-secret + bar: + - name: another-pull-secret +usage: | + {{ tuple . "bar" | include "helm-toolkit.snippets.kubernetes_image_pull_secrets" }} +return: | + imagePullSecrets: + - name: some-pull-secret + - name: another-pull-secret +*/}} + +{{- define "helm-toolkit.snippets.kubernetes_image_pull_secrets" -}} +{{- $envAll := index . 0 -}} +{{- $application := index . 1 -}} +{{- if ($envAll.Values.pod).image_pull_secrets }} +imagePullSecrets: +{{- if hasKey $envAll.Values.pod.image_pull_secrets $application }} +{{ index $envAll.Values.pod "image_pull_secrets" $application | toYaml | indent 2 }} +{{- end -}} +{{- if hasKey $envAll.Values.pod.image_pull_secrets "default" }} +{{ $envAll.Values.pod.image_pull_secrets.default | toYaml | indent 2 }} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/helm-toolkit/templates/snippets/_kubernetes_pod_rbac_roles.tpl b/helm-toolkit/templates/snippets/_kubernetes_pod_rbac_roles.tpl new file mode 100644 index 0000000000..90a7a65173 --- /dev/null +++ b/helm-toolkit/templates/snippets/_kubernetes_pod_rbac_roles.tpl @@ -0,0 +1,69 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "helm-toolkit.snippets.kubernetes_pod_rbac_roles" -}} +{{- $envAll := index . 0 -}} +{{- $deps := index . 1 -}} +{{- $saName := index . 2 | replace "_" "-" }} +{{- $saNamespace := index . 3 -}} +{{- $releaseName := $envAll.Release.Name }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $releaseName }}-{{ $saName }} + namespace: {{ $saNamespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $releaseName }}-{{ $saNamespace }}-{{ $saName }} +subjects: + - kind: ServiceAccount + name: {{ $saName }} + namespace: {{ $saNamespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $releaseName }}-{{ $saNamespace }}-{{ $saName }} + namespace: {{ $saNamespace }} +rules: + - apiGroups: + - "" + - extensions + - batch + - apps + verbs: + - get + - list + resources: + {{- range $k, $v := $deps -}} + {{ if eq $v "daemonsets" }} + - daemonsets + {{- end -}} + {{ if eq $v "jobs" }} + - jobs + {{- end -}} + {{ if or (eq $v "pods") (eq $v "daemonsets") (eq $v "jobs") }} + - pods + {{- end -}} + {{ if eq $v "services" }} + - services + - endpoints + {{- end -}} + {{ if eq $v "secrets" }} + - secrets + {{- end -}} + {{- end -}} +{{- end -}} diff --git a/helm-toolkit/templates/snippets/_kubernetes_pod_rbac_serviceaccount.tpl b/helm-toolkit/templates/snippets/_kubernetes_pod_rbac_serviceaccount.tpl new file mode 100644 index 0000000000..bc2045e5f2 --- /dev/null +++ b/helm-toolkit/templates/snippets/_kubernetes_pod_rbac_serviceaccount.tpl @@ -0,0 +1,75 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" -}} +{{- $envAll := index . 0 -}} +{{- $component := index . 1 -}} +{{- $saName := index . 2 -}} +{{- $saNamespace := $envAll.Release.Namespace }} +{{- $randomKey := randAlphaNum 32 }} +{{- $allNamespace := dict $randomKey "" }} + +{{- $_ := set $envAll.Values "__kubernetes_entrypoint_init_container" dict -}} +{{- $_ := set $envAll.Values.__kubernetes_entrypoint_init_container "deps" dict -}} +{{- if and ($envAll.Values.images.local_registry.active) (ne $component "image_repo_sync") -}} +{{- if eq $component "pod_dependency" -}} +{{- $_ := include "helm-toolkit.utils.merge" ( tuple $envAll.Values.__kubernetes_entrypoint_init_container.deps ( index $envAll.Values.pod_dependency ) $envAll.Values.dependencies.dynamic.common.local_image_registry ) -}} +{{- else -}} +{{- $_ := include "helm-toolkit.utils.merge" ( tuple $envAll.Values.__kubernetes_entrypoint_init_container.deps ( index $envAll.Values.dependencies.static $component ) $envAll.Values.dependencies.dynamic.common.local_image_registry ) -}} +{{- end -}} +{{- else -}} +{{- if eq $component "pod_dependency" -}} +{{- $_ := set $envAll.Values.__kubernetes_entrypoint_init_container "deps" ( index $envAll.Values.pod_dependency ) -}} +{{- else -}} +{{- $_ := set $envAll.Values.__kubernetes_entrypoint_init_container "deps" ( index $envAll.Values.dependencies.static $component ) -}} +{{- end -}} +{{- end -}} +{{- $deps := $envAll.Values.__kubernetes_entrypoint_init_container.deps }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ $saName }} + namespace: {{ $saNamespace }} +{{- if $envAll.Values.manifests.secret_registry }} +{{- if $envAll.Values.endpoints.oci_image_registry.auth.enabled }} +imagePullSecrets: + - name: {{ index $envAll.Values.secrets.oci_image_registry $envAll.Chart.Name }} +{{- end -}} +{{- end -}} +{{- range $k, $v := $deps -}} +{{- if eq $k "services" }} +{{- range $serv := $v }} +{{- $endpointMap := index $envAll.Values.endpoints $serv.service }} +{{- $endpointNS := $endpointMap.namespace | default $saNamespace }} +{{- if not (contains "services" ((index $allNamespace $endpointNS) | default "")) }} +{{- $_ := set $allNamespace $endpointNS (printf "%s%s" "services," ((index $allNamespace $endpointNS) | default "")) }} +{{- end -}} +{{- end -}} +{{- else if and (eq $k "jobs") $v }} +{{- $_ := set $allNamespace $saNamespace (printf "%s%s" "jobs," ((index $allNamespace $saNamespace) | default "")) }} +{{- else if and (eq $k "daemonset") $v }} +{{- $_ := set $allNamespace $saNamespace (printf "%s%s" "daemonsets," ((index $allNamespace $saNamespace) | default "")) }} +{{- else if and (eq $k "pod") $v }} +{{- $_ := set $allNamespace $saNamespace (printf "%s%s" "pods," ((index $allNamespace $saNamespace) | default "")) }} +{{- else if and (eq $k "secret") $v }} +{{- $_ := set $allNamespace $saNamespace (printf "%s%s" "secrets," ((index $allNamespace $saNamespace) | default "")) }} +{{- end -}} +{{- end -}} +{{- $_ := unset $allNamespace $randomKey }} +{{- range $ns, $vv := $allNamespace }} +{{- $resourceList := (splitList "," (trimSuffix "," $vv)) }} +{{- tuple $envAll $resourceList $saName $ns | include "helm-toolkit.snippets.kubernetes_pod_rbac_roles" }} +{{- end -}} +{{- end -}} diff --git a/helm-toolkit/templates/snippets/_kubernetes_pod_security_context.tpl b/helm-toolkit/templates/snippets/_kubernetes_pod_security_context.tpl new file mode 100644 index 0000000000..3a4fbaa8bc --- /dev/null +++ b/helm-toolkit/templates/snippets/_kubernetes_pod_security_context.tpl @@ -0,0 +1,67 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Renders securityContext for a Kubernetes pod. + For pod level, seurity context see here: https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.13/#podsecuritycontext-v1-core +examples: + - values: | + pod: + # NOTE: The 'user' key is deprecated, and will be removed shortly. + user: + myApp: + uid: 34356 + security_context: + myApp: + pod: + runAsNonRoot: true + usage: | + {{ dict "envAll" . "application" "myApp" | include "helm-toolkit.snippets.kubernetes_pod_security_context" }} + return: | + securityContext: + runAsUser: 34356 + runAsNonRoot: true + - values: | + pod: + security_context: + myApp: + pod: + runAsUser: 34356 + runAsNonRoot: true + usage: | + {{ dict "envAll" . "application" "myApp" | include "helm-toolkit.snippets.kubernetes_pod_security_context" }} + return: | + securityContext: + runAsNonRoot: true + runAsUser: 34356 +*/}} + +{{- define "helm-toolkit.snippets.kubernetes_pod_security_context" -}} +{{- $envAll := index . "envAll" -}} +{{- $application := index . "application" -}} +securityContext: +{{- if hasKey $envAll.Values.pod "user" }} +{{- if hasKey $envAll.Values.pod.user $application }} +{{- if hasKey ( index $envAll.Values.pod.user $application ) "uid" }} + runAsUser: {{ index $envAll.Values.pod.user $application "uid" }} +{{- end -}} +{{- end -}} +{{- end -}} +{{- if hasKey $envAll.Values.pod "security_context" }} +{{- if hasKey ( index $envAll.Values.pod.security_context ) $application }} +{{ toYaml ( index $envAll.Values.pod.security_context $application "pod" ) | indent 2 }} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/helm-toolkit/templates/snippets/_kubernetes_probes.tpl b/helm-toolkit/templates/snippets/_kubernetes_probes.tpl new file mode 100644 index 0000000000..7470760e03 --- /dev/null +++ b/helm-toolkit/templates/snippets/_kubernetes_probes.tpl @@ -0,0 +1,55 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Renders kubernetes liveness and readiness probes for containers +values: | + pod: + probes: + api: + default: + readiness: + enabled: true + params: + initialDelaySeconds: 30 + timeoutSeconds: 30 +usage: | + {{- define "probeTemplate" }} + httpGet: + path: /status + port: 9090 + {{- end }} + {{ dict "envAll" . "component" "api" "container" "default" "type" "readiness" "probeTemplate" (include "probeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" }} +return: | + readinessProbe: + httpGet: + path: /status + port: 9090 + initialDelaySeconds: 30 + timeoutSeconds: 30 +*/}} + +{{- define "helm-toolkit.snippets.kubernetes_probe" -}} +{{- $envAll := index . "envAll" -}} +{{- $component := index . "component" -}} +{{- $container := index . "container" -}} +{{- $type := index . "type" -}} +{{- $probeTemplate := index . "probeTemplate" -}} +{{- $probeOpts := index $envAll.Values.pod.probes $component $container $type -}} +{{- if $probeOpts.enabled -}} +{{- $probeOverides := index $probeOpts "params" | default dict -}} +{{ dict ( printf "%sProbe" $type ) (mergeOverwrite $probeTemplate $probeOverides ) | toYaml }} +{{- end -}} +{{- end -}} diff --git a/helm-toolkit/templates/snippets/_kubernetes_resources.tpl b/helm-toolkit/templates/snippets/_kubernetes_resources.tpl new file mode 100644 index 0000000000..24d30cf329 --- /dev/null +++ b/helm-toolkit/templates/snippets/_kubernetes_resources.tpl @@ -0,0 +1,53 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +Note: This function is deprecated and will be removed in the future. + +abstract: | + Renders kubernetes resource limits for pods +values: | + pod: + resources: + enabled: true + api: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + hugepages-1Gi: "1Gi" + +usage: | + {{ include "helm-toolkit.snippets.kubernetes_resources" ( tuple . .Values.pod.resources.api ) }} +return: | + resources: + limits: + cpu: "2000m" + memory: "1024Mi" + hugepages-1Gi: "1Gi" + requests: + cpu: "100m" + memory: "128Mi +*/}} + +{{- define "helm-toolkit.snippets.kubernetes_resources" -}} +{{- $envAll := index . 0 -}} +{{- $component := index . 1 -}} +{{- if $envAll.Values.pod.resources.enabled -}} +resources: +{{ toYaml $component | trim | indent 2 }} +{{- end -}} +{{- end -}} diff --git a/helm-toolkit/templates/snippets/_kubernetes_seccomp_annotation.tpl b/helm-toolkit/templates/snippets/_kubernetes_seccomp_annotation.tpl new file mode 100644 index 0000000000..555ffb051a --- /dev/null +++ b/helm-toolkit/templates/snippets/_kubernetes_seccomp_annotation.tpl @@ -0,0 +1,47 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Renders seccomp annotations for a list of containers driven by values.yaml. +values: | + pod: + seccomp: + myPodName: + myContainerName: localhost/mySeccomp + mySecondContainerName: localhost/secondProfile # optional + myThirdContainerName: localhost/thirdProfile # optional +usage: | + {{ dict "envAll" . "podName" "myPodName" "containerNames" (list "myContainerName" "mySecondContainerName" "myThirdContainerName") | include "helm-toolkit.snippets.kubernetes_seccomp_annotation" }} +return: | + container.seccomp.security.alpha.kubernetes.io/myContainerName: localhost/mySeccomp + container.seccomp.security.alpha.kubernetes.io/mySecondContainerName: localhost/secondProfile + container.seccomp.security.alpha.kubernetes.io/myThirdContainerName: localhost/thirdProfile +note: | + The number of container underneath is a variable arguments. It loops through + all the container names specified. +*/}} +{{- define "helm-toolkit.snippets.kubernetes_seccomp_annotation" -}} +{{- $envAll := index . "envAll" -}} +{{- $podName := index . "podName" -}} +{{- $containerNames := index . "containerNames" -}} +{{- if hasKey (index $envAll.Values.pod "seccomp") $podName -}} +{{- range $name := $containerNames -}} +{{- $seccompProfile := index $envAll.Values.pod.seccomp $podName $name -}} +{{- if $seccompProfile }} +container.seccomp.security.alpha.kubernetes.io/{{ $name }}: {{ $seccompProfile }} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/helm-toolkit/templates/snippets/_kubernetes_tolerations.tpl b/helm-toolkit/templates/snippets/_kubernetes_tolerations.tpl new file mode 100644 index 0000000000..e4af6a62a0 --- /dev/null +++ b/helm-toolkit/templates/snippets/_kubernetes_tolerations.tpl @@ -0,0 +1,45 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Renders kubernetes tolerations for pods +values: | + pod: + tolerations: + api: + enabled: true + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + - key: node-role.kubernetes.io/node + operator: Exists + +usage: | + {{ include "helm-toolkit.snippets.kubernetes_tolerations" ( tuple . .Values.pod.tolerations.api ) }} +return: | + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + - key: node-role.kubernetes.io/node + operator: Exists +*/}} + +{{- define "helm-toolkit.snippets.kubernetes_tolerations" -}} +{{- $envAll := index . 0 -}} +{{- $component := index . 1 -}} +{{- $pod := index $envAll.Values.pod.tolerations $component }} +tolerations: +{{ toYaml $pod.tolerations }} +{{- end -}} diff --git a/helm-toolkit/templates/snippets/_kubernetes_upgrades_daemonset.tpl b/helm-toolkit/templates/snippets/_kubernetes_upgrades_daemonset.tpl new file mode 100644 index 0000000000..69cee47216 --- /dev/null +++ b/helm-toolkit/templates/snippets/_kubernetes_upgrades_daemonset.tpl @@ -0,0 +1,33 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "helm-toolkit.snippets.kubernetes_upgrades_daemonset" -}} +{{- $envAll := index . 0 -}} +{{- $component := index . 1 -}} +{{- $upgradeMap := index $envAll.Values.pod.lifecycle.upgrades.daemonsets $component -}} +{{- $pod_replacement_strategy := $envAll.Values.pod.lifecycle.upgrades.daemonsets.pod_replacement_strategy -}} +{{- with $upgradeMap -}} +{{- if .enabled }} +minReadySeconds: {{ .min_ready_seconds }} +updateStrategy: + type: {{ $pod_replacement_strategy }} + {{- if $pod_replacement_strategy }} + {{- if eq $pod_replacement_strategy "RollingUpdate" }} + rollingUpdate: + maxUnavailable: {{ .max_unavailable }} + {{- end }} + {{- end }} +{{- end }} +{{- end -}} +{{- end -}} diff --git a/helm-toolkit/templates/snippets/_kubernetes_upgrades_deployment.tpl b/helm-toolkit/templates/snippets/_kubernetes_upgrades_deployment.tpl new file mode 100644 index 0000000000..be28cdb809 --- /dev/null +++ b/helm-toolkit/templates/snippets/_kubernetes_upgrades_deployment.tpl @@ -0,0 +1,27 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "helm-toolkit.snippets.kubernetes_upgrades_deployment" -}} +{{- $envAll := index . 0 -}} +{{- with $envAll.Values.pod.lifecycle.upgrades.deployments -}} +revisionHistoryLimit: {{ .revision_history }} +strategy: + type: {{ .pod_replacement_strategy }} + {{- if eq .pod_replacement_strategy "RollingUpdate" }} + rollingUpdate: + maxUnavailable: {{ .rolling_update.max_unavailable }} + maxSurge: {{ .rolling_update.max_surge }} + {{- end }} +{{- end -}} +{{- end -}} diff --git a/helm-toolkit/templates/snippets/_kubernetes_upgrades_statefulset.tpl b/helm-toolkit/templates/snippets/_kubernetes_upgrades_statefulset.tpl new file mode 100644 index 0000000000..f897023fee --- /dev/null +++ b/helm-toolkit/templates/snippets/_kubernetes_upgrades_statefulset.tpl @@ -0,0 +1,51 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Renders upgradeStrategy configuration for Kubernetes statefulsets. + See: https://kubernetes.io/docs/tutorials/stateful-application/basic-stateful-set/#updating-statefulsets + Types: + - RollingUpdate (default) + - OnDelete + Partitions: + - Stage updates to a statefulset by keeping pods at current version while + allowing mutations to statefulset's .spec.template +values: | + pod: + lifecycle: + upgrades: + statefulsets: + pod_replacement_strategy: RollingUpdate + partition: 2 +usage: | + {{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_statefulset" | indent 2 }} +return: | + updateStrategy: + type: RollingUpdate + rollingUpdate: + partition: 2 +*/}} + +{{- define "helm-toolkit.snippets.kubernetes_upgrades_statefulset" -}} +{{- $envAll := index . 0 -}} +{{- with $envAll.Values.pod.lifecycle.upgrades.statefulsets -}} +updateStrategy: + type: {{ .pod_replacement_strategy }} + {{ if .partition -}} + rollingUpdate: + partition: {{ .partition }} + {{- end -}} +{{- end -}} +{{- end -}} diff --git a/helm-toolkit/templates/snippets/_mon_host_from_k8s_ep.sh.tpl b/helm-toolkit/templates/snippets/_mon_host_from_k8s_ep.sh.tpl new file mode 100644 index 0000000000..fc74c6fb48 --- /dev/null +++ b/helm-toolkit/templates/snippets/_mon_host_from_k8s_ep.sh.tpl @@ -0,0 +1,68 @@ +{{- define "helm-toolkit.snippets.mon_host_from_k8s_ep" -}} +{{/* + +Inserts a bash function definition mon_host_from_k8s_ep() which can be used +to construct a mon_hosts value from the given namespaced endpoint. + +Usage (e.g. in _script.sh.tpl): + #!/bin/bash + + : "${NS:=ceph}" + : "${EP:=ceph-mon-discovery}" + + {{ include "helm-toolkit.snippets.mon_host_from_k8s_ep" . }} + + MON_HOST=$(mon_host_from_k8s_ep "$NS" "$EP") + + if [ -z "$MON_HOST" ]; then + # deal with failure + else + sed -i -e "s/^mon_host = /mon_host = $MON_HOST/" /etc/ceph/ceph.conf + fi +*/}} +{{` +# Construct a mon_hosts value from the given namespaced endpoint +# IP x.x.x.x with port p named "mon-msgr2" will appear as [v2:x.x.x.x/p/0] +# IP x.x.x.x with port q named "mon" will appear as [v1:x.x.x.x/q/0] +# IP x.x.x.x with ports p and q will appear as [v2:x.x.x.x/p/0,v1:x.x.x.x/q/0] +# The entries for all IPs will be joined with commas +mon_host_from_k8s_ep() { + local ns=$1 + local ep=$2 + + if [ -z "$ns" ] || [ -z "$ep" ]; then + return 1 + fi + + # We don't want shell expansion for the go-template expression + # shellcheck disable=SC2016 + kubectl get endpoints -n "$ns" "$ep" -o go-template=' + {{- $sep := "" }} + {{- range $_,$s := .subsets }} + {{- $v2port := 0 }} + {{- $v1port := 0 }} + {{- range $_,$port := index $s "ports" }} + {{- if (eq $port.name "mon-msgr2") }} + {{- $v2port = $port.port }} + {{- else if (eq $port.name "mon") }} + {{- $v1port = $port.port }} + {{- end }} + {{- end }} + {{- range $_,$address := index $s "addresses" }} + {{- $v2endpoint := printf "v2:%s:%d/0" $address.ip $v2port }} + {{- $v1endpoint := printf "v1:%s:%d/0" $address.ip $v1port }} + {{- if (and $v2port $v1port) }} + {{- printf "%s[%s,%s]" $sep $v2endpoint $v1endpoint }} + {{- $sep = "," }} + {{- else if $v2port }} + {{- printf "%s[%s]" $sep $v2endpoint }} + {{- $sep = "," }} + {{- else if $v1port }} + {{- printf "%s[%s]" $sep $v1endpoint }} + {{- $sep = "," }} + {{- end }} + {{- end }} + {{- end }}' +} +`}} +{{- end -}} diff --git a/helm-toolkit/templates/snippets/_prometheus_pod_annotations.tpl b/helm-toolkit/templates/snippets/_prometheus_pod_annotations.tpl new file mode 100644 index 0000000000..fec41f85d6 --- /dev/null +++ b/helm-toolkit/templates/snippets/_prometheus_pod_annotations.tpl @@ -0,0 +1,33 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# Appends annotations for configuring prometheus scrape jobs via pod +# annotations. The required annotations are: +# * `prometheus.io/scrape`: Only scrape pods that have a value of `true` +# * `prometheus.io/path`: If the metrics path is not `/metrics` override this. +# * `prometheus.io/port`: Scrape the pod on the indicated port instead of the +# pod's declared ports (default is a port-free target if none are declared). + +{{- define "helm-toolkit.snippets.prometheus_pod_annotations" -}} +{{- $config := index . 0 -}} +{{- if $config.scrape }} +prometheus.io/scrape: {{ $config.scrape | quote }} +{{- end }} +{{- if $config.path }} +prometheus.io/path: {{ $config.path | quote }} +{{- end }} +{{- if $config.port }} +prometheus.io/port: {{ $config.port | quote }} +{{- end }} +{{- end -}} diff --git a/helm-toolkit/templates/snippets/_prometheus_service_annotations.tpl b/helm-toolkit/templates/snippets/_prometheus_service_annotations.tpl new file mode 100644 index 0000000000..a827c4beff --- /dev/null +++ b/helm-toolkit/templates/snippets/_prometheus_service_annotations.tpl @@ -0,0 +1,35 @@ +{{/* +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# Appends annotations for configuring prometheus scrape endpoints via +# annotations. The required annotations are: +# * `prometheus.io/scrape`: Only scrape services that have a value of `true` +# * `prometheus.io/scheme`: If the metrics endpoint is secured then you will need +# to set this to `https` & most likely set the `tls_config` of the scrape config. +# * `prometheus.io/path`: If the metrics path is not `/metrics` override this. +# * `prometheus.io/port`: If the metrics are exposed on a different port to the +# service then set this appropriately. + +{{- define "helm-toolkit.snippets.prometheus_service_annotations" -}} +{{- $config := index . 0 -}} +{{- if $config.scrape }} +prometheus.io/scrape: {{ $config.scrape | quote }} +{{- end }} +{{- if $config.scheme }} +prometheus.io/scheme: {{ $config.scheme | quote }} +{{- end }} +{{- if $config.path }} +prometheus.io/path: {{ $config.path | quote }} +{{- end }} +{{- if $config.port }} +prometheus.io/port: {{ $config.port | quote }} +{{- end }} +{{- end -}} diff --git a/helm-toolkit/templates/snippets/_release_uuid.tpl b/helm-toolkit/templates/snippets/_release_uuid.tpl new file mode 100644 index 0000000000..253920b77f --- /dev/null +++ b/helm-toolkit/templates/snippets/_release_uuid.tpl @@ -0,0 +1,29 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Reneders an attonation key and value for a release +values: | + release_uuid: null +usage: | + {{ tuple . | include "helm-toolkit.snippets.release_uuid" }} +return: | + "openstackhelm.openstack.org/release_uuid": "" +*/}} + +{{- define "helm-toolkit.snippets.release_uuid" -}} +{{- $envAll := index . 0 -}} +"openstackhelm.openstack.org/release_uuid": {{ $envAll.Values.release_uuid | default "" | quote }} +{{- end -}} diff --git a/helm-toolkit/templates/snippets/_rgw_s3_admin_env_vars.tpl b/helm-toolkit/templates/snippets/_rgw_s3_admin_env_vars.tpl new file mode 100644 index 0000000000..a3169ce9ff --- /dev/null +++ b/helm-toolkit/templates/snippets/_rgw_s3_admin_env_vars.tpl @@ -0,0 +1,32 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "helm-toolkit.snippets.rgw_s3_admin_env_vars" }} +{{- $s3AdminSecret := .s3AdminSecret }} +- name: S3_ADMIN_USERNAME + valueFrom: + secretKeyRef: + name: {{ $s3AdminSecret }} + key: S3_ADMIN_USERNAME +- name: S3_ADMIN_ACCESS_KEY + valueFrom: + secretKeyRef: + name: {{ $s3AdminSecret }} + key: S3_ADMIN_ACCESS_KEY +- name: S3_ADMIN_SECRET_KEY + valueFrom: + secretKeyRef: + name: {{ $s3AdminSecret }} + key: S3_ADMIN_SECRET_KEY +{{- end }} diff --git a/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl b/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl new file mode 100644 index 0000000000..08521e0fe2 --- /dev/null +++ b/helm-toolkit/templates/snippets/_rgw_s3_bucket_user_env_vars_rook.tpl @@ -0,0 +1,28 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "helm-toolkit.snippets.rgw_s3_bucket_user_env_vars_rook" }} +{{- range $s3Bucket := .Values.storage.s3.buckets }} +- name: {{ printf "%s_S3_ACCESS_KEY" ($s3Bucket.client | replace "-" "_" | upper) }} + valueFrom: + secretKeyRef: + name: {{ $s3Bucket.name }} + key: AWS_ACCESS_KEY_ID +- name: {{ printf "%s_S3_SECRET_KEY" ($s3Bucket.client | replace "-" "_" | upper) }} + valueFrom: + secretKeyRef: + name: {{ $s3Bucket.name }} + key: AWS_SECRET_ACCESS_KEY +{{- end }} +{{- end }} diff --git a/helm-toolkit/templates/snippets/_rgw_s3_secret_creds.tpl b/helm-toolkit/templates/snippets/_rgw_s3_secret_creds.tpl new file mode 100644 index 0000000000..a611a5e757 --- /dev/null +++ b/helm-toolkit/templates/snippets/_rgw_s3_secret_creds.tpl @@ -0,0 +1,29 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "helm-toolkit.snippets.rgw_s3_secret_creds" }} +{{- range $client, $config := .Values.storage.s3.clients -}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ printf "%s-s3-user-secret" ( $client | replace "_" "-" | lower ) }} +type: Opaque +data: +{{- range $key, $value := $config.auth }} + {{ $key | upper }}: {{ $value | toString | b64enc}} +{{- end }} + +{{ end }} +{{- end }} diff --git a/helm-toolkit/templates/snippets/_rgw_s3_user_env_vars.tpl b/helm-toolkit/templates/snippets/_rgw_s3_user_env_vars.tpl new file mode 100644 index 0000000000..a3dd4314bb --- /dev/null +++ b/helm-toolkit/templates/snippets/_rgw_s3_user_env_vars.tpl @@ -0,0 +1,34 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "helm-toolkit.snippets.rgw_s3_user_env_vars" }} +{{- range $client, $user := .Values.storage.s3.clients }} +{{- $s3secret := printf "%s-s3-user-secret" ( $client | replace "_" "-" | lower ) }} +- name: {{ printf "%s_S3_USERNAME" ($client | replace "-" "_" | upper) }} + valueFrom: + secretKeyRef: + name: {{ $s3secret }} + key: USERNAME +- name: {{ printf "%s_S3_ACCESS_KEY" ($client | replace "-" "_" | upper) }} + valueFrom: + secretKeyRef: + name: {{ $s3secret }} + key: ACCESS_KEY +- name: {{ printf "%s_S3_SECRET_KEY" ($client | replace "-" "_" | upper) }} + valueFrom: + secretKeyRef: + name: {{ $s3secret }} + key: SECRET_KEY +{{- end }} +{{- end }} diff --git a/helm-toolkit/templates/snippets/_service_params.tpl b/helm-toolkit/templates/snippets/_service_params.tpl new file mode 100644 index 0000000000..6233a93556 --- /dev/null +++ b/helm-toolkit/templates/snippets/_service_params.tpl @@ -0,0 +1,61 @@ +{{/* +Copyright 2017 The Openstack-Helm Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{/* +abstract: | + Inserts kubernetes service parameters from values as is. +values: | + network: + serviceExample: + service: + type: loadBalancer + loadBalancerIP: 1.1.1.1 +usage: | + --- + apiVersion: v1 + kind: Service + metadata: + name: 'serviceExample' + spec: + ports: + - name: s-example + port: 1111 + {{ .Values.network.serviceExample | include "helm-toolkit.snippets.service_params" | indent 2 }} +return: | + type: loadBalancer + loadBalancerIP: 1.1.1.1 +*/}} + +{{- define "helm-toolkit.snippets.service_params" }} +{{- $serviceParams := dict }} +{{- if hasKey . "service" }} +{{- $serviceParams = .service }} +{{- end }} +{{- if hasKey . "node_port" }} +{{- if hasKey .node_port "enabled" }} +{{- if .node_port.enabled }} +{{- $_ := set $serviceParams "type" "NodePort" }} +{{- end }} +{{- end }} +{{- end }} +{{- if hasKey . "external_policy_local" }} +{{- if .external_policy_local }} +{{- $_ := set $serviceParams "externalTrafficPolicy" "Local" }} +{{- end }} +{{- end }} +{{- if $serviceParams }} +{{- $serviceParams | toYaml }} +{{- end }} +{{- end }} diff --git a/helm-toolkit/templates/snippets/_tls_volume.tpl b/helm-toolkit/templates/snippets/_tls_volume.tpl new file mode 100644 index 0000000000..41fe3d96db --- /dev/null +++ b/helm-toolkit/templates/snippets/_tls_volume.tpl @@ -0,0 +1,47 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{/* +abstract: | + Renders a secret volume for tls. + + Dictionary Parameters: + enabled: boolean check if you want to conditional disable this snippet (optional) + name: name of the volume (required) + secretName: name of a kuberentes/tls secret, if not specified, use the volume name (optional) + +values: | + manifests: + certificates: true + +usage: | + {{- $opts := dict "enabled" "true" "name" "glance-tls-api" -}} + {{- $opts | include "helm-toolkit.snippets.tls_volume" -}} + +return: | + - name: glance-tls-api + secret: + secretName: glance-tls-api + defaultMode: 292 +*/}} +{{- define "helm-toolkit.snippets.tls_volume" }} +{{- $enabled := index . "enabled" -}} +{{- $name := index . "name" -}} +{{- $secretName := index . "secretName" | default $name -}} +{{- if and $enabled (ne $name "") }} +- name: {{ $name }} + secret: + secretName: {{ $secretName }} + defaultMode: 292 +{{- end }} +{{- end }} diff --git a/helm-toolkit/templates/snippets/_tls_volume_mount.tpl b/helm-toolkit/templates/snippets/_tls_volume_mount.tpl new file mode 100644 index 0000000000..9cfa81950b --- /dev/null +++ b/helm-toolkit/templates/snippets/_tls_volume_mount.tpl @@ -0,0 +1,82 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{/* +abstract: | + Renders a volume mount for TLS key, cert and CA. + + Dictionary Parameters: + enabled: boolean check if you want to conditional disable this snippet (optional) + name: name that of the volume and should match the volume name (required) + path: path to place tls.crt tls.key ca.crt, do not suffix with '/' (required) + certs: a tuple containing a nonempty subset of {tls.crt, tls.key, ca.crt}. + the default is the full set. (optional) + +values: | + manifests: + certificates: true + +usage: | + {{- $opts := dict "enabled" .Values.manifests.certificates "name" "glance-tls-api" "path" "/etc/glance/certs" -}} + {{- $opts | include "helm-toolkit.snippets.tls_volume_mount" -}} + +return: | + - name: glance-tls-api + mountPath: /etc/glance/certs/tls.crt + subPath: tls.crt + readOnly: true + - name: glance-tls-api + mountPath: /etc/glance/certs/tls.key + subPath: tls.key + readOnly: true + - name: glance-tls-api + mountPath: /etc/glance/certs/ca.crt + subPath: ca.crt + readOnly: true + +abstract: | + This mounts a specific issuing CA only for service validation + +usage: | + {{- $opts := dict "enabled" .Values.manifests.certificates "name" "glance-tls-api" "ca" true -}} + {{- $opts | include "helm-toolkit.snippets.tls_volume_mount" -}} + +return: | + - name: glance-tls-api + mountPath: /etc/ssl/certs/openstack-helm.crt + subPath: ca.crt + readOnly: true +*/}} +{{- define "helm-toolkit.snippets.tls_volume_mount" }} +{{- $enabled := index . "enabled" -}} +{{- $name := index . "name" -}} +{{- $path := index . "path" | default "" -}} +{{- $certs := index . "certs" | default ( tuple "tls.crt" "tls.key" "ca.crt" ) }} +{{- if $enabled }} +{{- if and (eq $path "") (ne $name "") }} +- name: {{ $name }} + mountPath: "/etc/ssl/certs/openstack-helm.crt" + subPath: ca.crt + readOnly: true +{{- else }} +{{- if ne $name "" }} +{{- range $key, $value := $certs }} +- name: {{ $name }} + mountPath: {{ printf "%s/%s" $path $value }} + subPath: {{ $value }} + readOnly: true +{{- end }} +{{- end }} +{{- end }} +{{- end }} +{{- end }} diff --git a/helm-toolkit/templates/snippets/_values_template_renderer.tpl b/helm-toolkit/templates/snippets/_values_template_renderer.tpl new file mode 100644 index 0000000000..6e9d5a1844 --- /dev/null +++ b/helm-toolkit/templates/snippets/_values_template_renderer.tpl @@ -0,0 +1,87 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Renders out configuration sections into a format suitable for incorporation + into a config-map. Allowing various forms of input to be rendered out as + appropriate. +values: | + conf: + inputs: + - foo + - bar + some: + config_to_render: | + #We can use all of gotpl here: eg macros, ranges etc. + {{ include "helm-toolkit.utils.joinListWithComma" .Values.conf.inputs }} + config_to_complete: + #here we can fill out params, but things need to be valid yaml as input + '{{ .Release.Name }}': '{{ printf "%s-%s" .Release.Namespace "namespace" }}' + static_config: + #this is just passed though as yaml to the configmap + foo: bar +usage: | + {{- $envAll := . }} + --- + apiVersion: v1 + kind: ConfigMap + metadata: + name: application-etc + data: + {{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.some.config_to_render "key" "config_to_render.conf") | indent 2 }} + {{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.some.config_to_complete "key" "config_to_complete.yaml") | indent 2 }} + {{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.some.static_config "key" "static_config.yaml") | indent 2 }} +return: | + --- + apiVersion: v1 + kind: ConfigMap + metadata: + name: application-etc + data: + config_to_render.conf: | + #We can use all of gotpl here: eg macros, ranges etc. + foo,bar + + config_to_complete.yaml: | + 'RELEASE-NAME': 'default-namespace' + + static_config.yaml: | + foo: bar +*/}} + +{{- define "helm-toolkit.snippets.values_template_renderer" -}} +{{- $envAll := index . "envAll" -}} +{{- $template := index . "template" -}} +{{- $key := index . "key" -}} +{{- $format := index . "format" | default "configMap" -}} +{{- with $envAll -}} +{{- $templateRendered := tpl ( $template | toYaml ) . }} +{{- if eq $format "Secret" }} +{{- if hasPrefix "|\n" $templateRendered }} +{{ $key }}: {{ regexReplaceAllLiteral "\n " ( $templateRendered | trimPrefix "|\n" | trimPrefix " " ) "\n" | b64enc }} +{{- else }} +{{ $key }}: {{ $templateRendered | b64enc }} +{{- end -}} +{{- else }} +{{- if hasPrefix "|\n" $templateRendered }} +{{ $key }}: | +{{ regexReplaceAllLiteral "\n " ( $templateRendered | trimPrefix "|\n" | trimPrefix " " ) "\n" | indent 2 }} +{{- else }} +{{ $key }}: | +{{ $templateRendered | indent 2 }} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/helm-toolkit/templates/tls/_tls_generate_certs.tpl b/helm-toolkit/templates/tls/_tls_generate_certs.tpl new file mode 100644 index 0000000000..6d617a182e --- /dev/null +++ b/helm-toolkit/templates/tls/_tls_generate_certs.tpl @@ -0,0 +1,94 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Produces a certificate from a certificate authority. If the "encode" parameter + is true, base64 encode the values for inclusion in a Kubernetes secret. +values: | + test: + hosts: + names: + - barbican.openstackhelm.example + - barbican.openstack.svc.cluster.local + ips: + - 127.0.0.1 + - 192.168.0.1 + life: 3 + # Use ca.crt and ca.key to build a customized ca, if they are provided. + # Use hosts.names[0] and life to auto-generate a ca, if ca is not provided. + ca: + crt: | + + key: | + +usage: | + {{ include "helm-toolkit.utils.tls_generate_certs" (dict "params" .Values.test) }} +return: | + ca: | + + crt: | + + exp: 2018-09-01T10:56:07.895392915-00:00 + key: | + +*/}} + +{{- define "helm-toolkit.utils.tls_generate_certs" -}} +{{- $params := index . "params" -}} +{{- $encode := index . "encode" | default false -}} +{{- $local := dict -}} + +{{- $_hosts := $params.hosts.names | default list }} +{{- if kindIs "string" $params.hosts.names }} +{{- $_ := set $local "certHosts" (list $params.hosts.names) }} +{{- else }} +{{- $_ := set $local "certHosts" $_hosts }} +{{- end }} + +{{- $_ips := $params.hosts.ips | default list }} +{{- if kindIs "string" $params.hosts.ips }} +{{- $_ := set $local "certIps" (list $params.hosts.ips) }} +{{- else }} +{{- $_ := set $local "certIps" $_ips }} +{{- end }} + +{{- if hasKey $params "ca" }} +{{- if and (hasKey $params.ca "crt") (hasKey $params.ca "key") }} +{{- $ca := buildCustomCert ($params.ca.crt | b64enc ) ($params.ca.key | b64enc ) }} +{{- $_ := set $local "ca" $ca }} +{{- end }} +{{- else }} +{{- $ca := genCA (first $local.certHosts) (int $params.life) }} +{{- $_ := set $local "ca" $ca }} +{{- end }} + +{{- $expDate := date_in_zone "2006-01-02T15:04:05Z07:00" ( date_modify (printf "+%sh" (mul $params.life 24 |toString)) now ) "UTC" }} +{{- $rawCert := genSignedCert (first $local.certHosts) ($local.certIps) ($local.certHosts) (int $params.life) $local.ca }} +{{- $certificate := dict -}} +{{- if $encode -}} +{{- $_ := b64enc $rawCert.Cert | set $certificate "crt" -}} +{{- $_ := b64enc $rawCert.Key | set $certificate "key" -}} +{{- $_ := b64enc $local.ca.Cert | set $certificate "ca" -}} +{{- $_ := b64enc $local.ca.Key | set $certificate "caKey" -}} +{{- $_ := b64enc $expDate | set $certificate "exp" -}} +{{- else -}} +{{- $_ := set $certificate "crt" $rawCert.Cert -}} +{{- $_ := set $certificate "key" $rawCert.Key -}} +{{- $_ := set $certificate "ca" $local.ca.Cert -}} +{{- $_ := set $certificate "caKey" $local.ca.Key -}} +{{- $_ := set $certificate "exp" $expDate -}} +{{- end -}} +{{- $certificate | toYaml }} +{{- end -}} diff --git a/helm-toolkit/templates/utils/_comma_joined_service_list.tpl b/helm-toolkit/templates/utils/_comma_joined_service_list.tpl new file mode 100644 index 0000000000..e26501f803 --- /dev/null +++ b/helm-toolkit/templates/utils/_comma_joined_service_list.tpl @@ -0,0 +1,46 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Returns a comma separated list of namespace:service pairs. +values: | + dependencies: + static: + api: + services: + - endpoint: internal + service: oslo_cache + - endpoint: internal + service: oslo_db + endpoints: + oslo_db: + namespace: foo + hosts: + default: mariadb + oslo_cache: + namespace: bar + hosts: + default: memcache +usage: | + {{ tuple .Values.dependencies.static.api.services . | include "helm-toolkit.utils.comma_joined_service_list" }} +return: | + bar:memcache,foo:mariadb +*/}} + +{{- define "helm-toolkit.utils.comma_joined_service_list" -}} +{{- $deps := index . 0 -}} +{{- $envAll := index . 1 -}} +{{- range $k, $v := $deps -}}{{- if $k -}},{{- end -}}{{ tuple $v.service $v.endpoint $envAll | include "helm-toolkit.endpoints.service_name_endpoint_with_namespace_lookup" }}{{- end -}} +{{- end -}} diff --git a/helm-toolkit/templates/utils/_configmap_templater.tpl b/helm-toolkit/templates/utils/_configmap_templater.tpl new file mode 100644 index 0000000000..7095c19373 --- /dev/null +++ b/helm-toolkit/templates/utils/_configmap_templater.tpl @@ -0,0 +1,30 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "helm-toolkit.utils.configmap_templater" }} +{{- $keyRoot := index . 0 -}} +{{- $configTemplate := index . 1 -}} +{{- $context := index . 2 -}} +{{ if $keyRoot.override -}} +{{ $keyRoot.override | indent 4 }} +{{- else -}} +{{- if $keyRoot.prefix -}} +{{ $keyRoot.prefix | indent 4 }} +{{- end }} +{{ tuple $configTemplate $context | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} +{{- if $keyRoot.append -}} +{{ $keyRoot.append | indent 4 }} +{{- end }} +{{- end -}} diff --git a/helm-toolkit/templates/utils/_daemonset_overrides.tpl b/helm-toolkit/templates/utils/_daemonset_overrides.tpl new file mode 100644 index 0000000000..40359f0f44 --- /dev/null +++ b/helm-toolkit/templates/utils/_daemonset_overrides.tpl @@ -0,0 +1,269 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "helm-toolkit.utils.daemonset_overrides" }} + {{- $daemonset := index . 0 }} + {{- $daemonset_yaml := index . 1 }} + {{- $configmap_include := index . 2 }} + {{- $configmap_name := index . 3 }} + {{- $context := index . 4 }} + {{- $_ := unset $context ".Files" }} + {{- $daemonset_root_name := printf (print $context.Chart.Name "_" $daemonset) }} + {{- $_ := set $context.Values "__daemonset_list" list }} + {{- $_ := set $context.Values "__default" dict }} + {{- if hasKey $context.Values.conf "overrides" }} + {{- range $key, $val := $context.Values.conf.overrides }} + + {{- if eq $key $daemonset_root_name }} + {{- range $type, $type_data := . }} + + {{- if eq $type "hosts" }} + {{- range $host_data := . }} + {{/* dictionary that will contain all info needed to generate this + iteration of the daemonset */}} + {{- $current_dict := dict }} + + {{/* set daemonset name */}} + {{/* Note: long hostnames can cause the 63 char name limit to be + exceeded. Truncate the hostname if hostname > 20 char */}} + {{- if gt (len $host_data.name) 20 }} + {{- $_ := set $current_dict "name" (substr 0 20 $host_data.name) }} + {{- else }} + {{- $_ := set $current_dict "name" $host_data.name }} + {{- end }} + + {{/* apply overrides */}} + {{- $override_conf_copy := $host_data.conf }} + {{/* Deep copy to prevent https://storyboard.openstack.org/#!/story/2005936 */}} + {{- $root_conf_copy := omit ($context.Values.conf | toYaml | fromYaml) "overrides" }} + {{- $merged_dict := mergeOverwrite $root_conf_copy $override_conf_copy }} + {{- $root_conf_copy2 := dict "conf" $merged_dict }} + {{- $context_values := omit (omit ($context.Values | toYaml | fromYaml) "conf") "__daemonset_list" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2 }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $current_dict "nodeData" $root_conf_copy4 }} + + {{/* Schedule to this host explicitly. */}} + {{- $nodeSelector_dict := dict }} + + {{- $_ := set $nodeSelector_dict "key" "kubernetes.io/hostname" }} + {{- $_ := set $nodeSelector_dict "operator" "In" }} + + {{- $values_list := list $host_data.name }} + {{- $_ := set $nodeSelector_dict "values" $values_list }} + + {{- $list_aggregate := list $nodeSelector_dict }} + {{- $_ := set $current_dict "matchExpressions" $list_aggregate }} + + {{/* store completed daemonset entry/info into global list */}} + {{- $list_aggregate := append $context.Values.__daemonset_list $current_dict }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + + {{- end }} + {{- end }} + + {{- if eq $type "labels" }} + {{- $_ := set $context.Values "__label_list" . }} + {{- range $label_data := . }} + {{/* dictionary that will contain all info needed to generate this + iteration of the daemonset. */}} + {{- $_ := set $context.Values "__current_label" dict }} + + {{/* set daemonset name */}} + {{- $_ := set $context.Values.__current_label "name" $label_data.label.key }} + + {{/* apply overrides */}} + {{- $override_conf_copy := $label_data.conf }} + {{/* Deep copy to prevent https://storyboard.openstack.org/#!/story/2005936 */}} + {{- $root_conf_copy := omit ($context.Values.conf | toYaml | fromYaml) "overrides" }} + {{- $merged_dict := mergeOverwrite $root_conf_copy $override_conf_copy }} + {{- $root_conf_copy2 := dict "conf" $merged_dict }} + {{- $context_values := omit (omit ($context.Values | toYaml | fromYaml) "conf") "__daemonset_list" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2 }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__current_label "nodeData" $root_conf_copy4 }} + + {{/* Schedule to the provided label value(s) */}} + {{- $label_dict := omit $label_data.label "NULL" }} + {{- $_ := set $label_dict "operator" "In" }} + {{- $list_aggregate := list $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + + {{/* Do not schedule to other specified labels, with higher + precedence as the list position increases. Last defined label + is highest priority. */}} + {{- $other_labels := without $context.Values.__label_list $label_data }} + {{- range $label_data2 := $other_labels }} + {{- $label_dict := omit $label_data2.label "NULL" }} + + {{- $_ := set $label_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__current_label.matchExpressions $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + {{- end }} + {{- $_ := set $context.Values "__label_list" $other_labels }} + + {{/* Do not schedule to any other specified hosts */}} + {{- range $type, $type_data := $val }} + {{- if eq $type "hosts" }} + {{- range $host_data := . }} + {{- $label_dict := dict }} + + {{- $_ := set $label_dict "key" "kubernetes.io/hostname" }} + {{- $_ := set $label_dict "operator" "NotIn" }} + + {{- $values_list := list $host_data.name }} + {{- $_ := set $label_dict "values" $values_list }} + + {{- $list_aggregate := append $context.Values.__current_label.matchExpressions $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + {{- end }} + {{- end }} + {{- end }} + + {{/* store completed daemonset entry/info into global list */}} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__current_label }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + {{- $_ := unset $context.Values "__current_label" }} + + {{- end }} + {{- end }} + {{- end }} + + {{/* scheduler exceptions for the default daemonset */}} + {{- $_ := set $context.Values.__default "matchExpressions" list }} + + {{- range $type, $type_data := . }} + {{/* Do not schedule to other specified labels */}} + {{- if eq $type "labels" }} + {{- range $label_data := . }} + {{- $default_dict := omit $label_data.label "NULL" }} + + {{- $_ := set $default_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__default.matchExpressions $default_dict }} + {{- $_ := set $context.Values.__default "matchExpressions" $list_aggregate }} + {{- end }} + {{- end }} + {{/* Do not schedule to other specified hosts */}} + {{- if eq $type "hosts" }} + {{- range $host_data := . }} + {{- $default_dict := dict }} + + {{- $_ := set $default_dict "key" "kubernetes.io/hostname" }} + {{- $_ := set $default_dict "operator" "NotIn" }} + + {{- $values_list := list $host_data.name }} + {{- $_ := set $default_dict "values" $values_list }} + + {{- $list_aggregate := append $context.Values.__default.matchExpressions $default_dict }} + {{- $_ := set $context.Values.__default "matchExpressions" $list_aggregate }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + + {{/* generate the default daemonset */}} + + {{/* set name */}} + {{- $_ := set $context.Values.__default "name" "default" }} + + {{/* no overrides apply, so copy as-is */}} + {{- $root_conf_copy1 := omit $context.Values.conf "overrides" }} + {{- $root_conf_copy2 := dict "conf" $root_conf_copy1 }} + {{- $context_values := omit $context.Values "conf" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2 }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__default "nodeData" $root_conf_copy4 }} + + {{/* add to global list */}} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__default }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + + {{- range $current_dict := $context.Values.__daemonset_list }} + + {{- $context_novalues := omit $context "Values" }} + {{- $merged_dict := mergeOverwrite $context_novalues $current_dict.nodeData }} + {{- $_ := set $current_dict "nodeData" $merged_dict }} + {{/* Deep copy original daemonset_yaml */}} + {{- $_ := set $context.Values "__daemonset_yaml" ($daemonset_yaml | toYaml | fromYaml) }} + + {{/* name needs to be a DNS-1123 compliant name. Ensure lower case */}} + {{- $name_format1 := printf (print $daemonset_root_name "-" $current_dict.name) | lower }} + {{/* labels may contain underscores which would be invalid here, so we replace them with dashes + there may be other valid label names which would make for an invalid DNS-1123 name + but these will be easier to handle in future with sprig regex* functions + (not availabile in helm 2.5.1) */}} + {{- $name_format2 := $name_format1 | replace "_" "-" }} + {{/* To account for the case where the same label is defined multiple times in overrides + (but with different label values), we add a sha of the scheduling data to ensure + name uniqueness */}} + {{- $_ := set $current_dict "dns_1123_name" dict }} + {{- if hasKey $current_dict "matchExpressions" }} + {{- $_ := set $current_dict "dns_1123_name" (printf (print $name_format2 "-" ($current_dict.matchExpressions | quote | sha256sum | trunc 8))) }} + {{- else }} + {{- $_ := set $current_dict "dns_1123_name" $name_format2 }} + {{- end }} + + {{/* set daemonset metadata name */}} + {{- if not $context.Values.__daemonset_yaml.metadata }}{{- $_ := set $context.Values.__daemonset_yaml "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.metadata.name }}{{- $_ := set $context.Values.__daemonset_yaml.metadata "name" dict }}{{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.metadata "name" $current_dict.dns_1123_name }} + + {{/* cross-reference configmap name to container volume definitions */}} + {{- $_ := set $context.Values "__volume_list" list }} + {{- range $current_volume := $context.Values.__daemonset_yaml.spec.template.spec.volumes }} + {{- $_ := set $context.Values "__volume" $current_volume }} + {{- if hasKey $context.Values.__volume "secret" }} + {{- if eq $context.Values.__volume.secret.secretName $configmap_name }} + {{- $_ := set $context.Values.__volume.secret "secretName" $current_dict.dns_1123_name }} + {{- end }} + {{- end }} + {{- $updated_list := append $context.Values.__volume_list $context.Values.__volume }} + {{- $_ := set $context.Values "__volume_list" $updated_list }} + {{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "volumes" $context.Values.__volume_list }} + + + {{/* populate scheduling restrictions */}} + {{- if hasKey $current_dict "matchExpressions" }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "affinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity "nodeAffinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity "requiredDuringSchedulingIgnoredDuringExecution" dict }}{{- end }} + {{- $match_exprs := dict }} + {{- $_ := set $match_exprs "matchExpressions" $current_dict.matchExpressions }} + {{- $appended_match_expr := list $match_exprs }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" $appended_match_expr }} + {{- end }} + + {{/* input value hash for current set of values overrides */}} + {{- if not $context.Values.__daemonset_yaml.spec }}{{- $_ := set $context.Values.__daemonset_yaml "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template }}{{- $_ := set $context.Values.__daemonset_yaml.spec "template" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata.annotations }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata "annotations" dict }}{{- end }} + {{- $cmap := list $current_dict.dns_1123_name $current_dict.nodeData | include $configmap_include }} + {{- $values_hash := $cmap | quote | sha256sum }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-etc-hash" $values_hash }} + + {{/* generate configmap */}} +--- +{{ $cmap }} + {{/* generate daemonset yaml */}} +--- +{{ $context.Values.__daemonset_yaml | toYaml }} + {{- end }} +{{- end }} diff --git a/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl b/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl new file mode 100644 index 0000000000..bdb28c3312 --- /dev/null +++ b/helm-toolkit/templates/utils/_daemonset_overrides_root.tpl @@ -0,0 +1,279 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* + +The helm-toolkit.utils.daemonset_overrides function have some limitations: + + * it allows to override only conf values specifid in configmap-etc + * it doesn't allow to override values for daemonsets passed via env variables + or via damoenset definition. As result it is impossible to have mixed + deployment when one compute is configured with dpdk while other not. + * it is impossible to override interface names/other information stored in + -bin configmap + * It allows to schedule on both hosts and labels, which adds some + uncertainty + +This implementation is intended to handle those limitations: + + * it allows to schedule only based on labels + * it creates -bin per daemonset override + * it allows to override values when rendering daemonsets + + It picks data from the following structure: + + .Values: + overrides: + mychart_mydaemonset: + labels: + label::value: + values: + override_root_option: override_root_value + conf: + ovs_dpdk: + enabled: true + neutron: + DEFAULT: + foo: bar + +*/}} + +{{- define "helm-toolkit.utils.daemonset_overrides_root" }} + {{- $daemonset := index . 0 }} + {{- $daemonSetTemplateName := index . 1 }} + {{ $serviceAccountName := index . 2 }} + {{- $configmap_include := index . 3 }} + {{- $configmap_name := index . 4 }} + {{- $configbin_include := index . 5 }} + {{- $configbin_name := index . 6 }} + {{- $context := index . 7 }} + + {{- $_ := unset $context ".Files" }} + {{- $daemonset_root_name := printf (print $context.Chart.Name "_" $daemonset) }} + {{- $_ := set $context.Values "__daemonset_list" list }} + {{- $_ := set $context.Values "__default" dict }} + + {{- $default_enabled := true }} + {{- if hasKey $context.Values "overrides" }} + {{- range $key, $val := $context.Values.overrides }} + + {{- if eq $key $daemonset_root_name }} + {{- range $type, $type_data := . }} + {{- if eq $type "overrides_default" }} + {{- $default_enabled = $type_data }} + {{- end }} + + {{- if eq $type "labels" }} + {{- $_ := set $context.Values "__label_dict" . }} + {{- range $lname, $ldata := . }} + {{ $label_name := (split "::" $lname)._0 }} + {{ $label_value := (split "::" $lname)._1 }} + {{/* dictionary that will contain all info needed to generate this + iteration of the daemonset. */}} + {{- $_ := set $context.Values "__current_label" dict }} + + {{/* set daemonset name */}} + {{- $_ := set $context.Values.__current_label "name" $label_name }} + + {{/* set daemonset metadata annotation */}} + {{- $_ := set $context.Values.__current_label "daemonset_override" $lname }} + + {{/* apply overrides */}} + + + {{- $override_root_copy := $ldata.values }} + {{/* Deep copy to prevent https://storyboard.openstack.org/#!/story/2005936 */}} + {{- $root_copy := omit ($context.Values | toYaml | fromYaml) "overrides" }} + {{- $merged_dict := mergeOverwrite $root_copy $override_root_copy }} + + {{- $root_conf_copy2 := dict "values" $merged_dict }} + {{- $context_values := omit (omit ($context.Values | toYaml | fromYaml) "values") "__daemonset_list" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2.values }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__current_label "nodeData" $root_conf_copy4 }} + + + {{/* Schedule to the provided label value(s) */}} + {{- $label_dict := dict "key" $label_name }} + {{- $_ := set $label_dict "values" (list $label_value) }} + {{- $_ := set $label_dict "operator" "In" }} + {{- $list_aggregate := list $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + + {{/* Do not schedule to other specified labels, with higher + precedence as the list position increases. Last defined label + is highest priority. */}} + {{- $other_labels := omit $context.Values.__label_dict $lname }} + {{- range $lname2, $ldata2 := $other_labels }} + {{ $label_name2 := (split "::" $lname2)._0 }} + {{ $label_value2 := (split "::" $lname2)._1 }} + + {{- $label_dict := dict "key" $label_name2 }} + {{- $_ := set $label_dict "values" (list $label_value2) }} + {{- $_ := set $label_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__current_label.matchExpressions $label_dict }} + {{- $_ := set $context.Values.__current_label "matchExpressions" $list_aggregate }} + {{- end }} + + {{/* store completed daemonset entry/info into global list */}} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__current_label }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + {{- $_ := unset $context.Values "__current_label" }} + + {{- end }} + {{- end }} + {{- end }} + + {{/* scheduler exceptions for the default daemonset */}} + {{- $_ := set $context.Values.__default "matchExpressions" list }} + + {{- range $type, $type_data := . }} + {{/* Do not schedule to other specified labels */}} + {{- if eq $type "labels" }} + {{- range $lname, $ldata := . }} + {{ $label_name := (split "::" $lname)._0 }} + {{ $label_value := (split "::" $lname)._1 }} + + {{- $default_dict := dict "key" $label_name }} + {{- $_ := set $default_dict "values" (list $label_value) }} + {{- $_ := set $default_dict "operator" "NotIn" }} + + {{- $list_aggregate := append $context.Values.__default.matchExpressions $default_dict }} + {{- $_ := set $context.Values.__default "matchExpressions" $list_aggregate }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + {{- end }} + + {{/* generate the default daemonset */}} + + {{/* set name */}} + {{- $_ := set $context.Values.__default "name" "default" }} + + {{/* no overrides apply, so copy as-is */}} + {{- $root_conf_copy1 := omit $context.Values.conf "overrides" }} + {{- $root_conf_copy2 := dict "conf" $root_conf_copy1 }} + {{- $context_values := omit $context.Values "conf" }} + {{- $root_conf_copy3 := mergeOverwrite $context_values $root_conf_copy2 }} + {{- $root_conf_copy4 := dict "Values" $root_conf_copy3 }} + {{- $_ := set $context.Values.__default "nodeData" $root_conf_copy4 }} + + {{/* add to global list */}} + {{- if $default_enabled }} + {{- $list_aggregate := append $context.Values.__daemonset_list $context.Values.__default }} + {{- $_ := set $context.Values "__daemonset_list" $list_aggregate }} + {{- end }} + + {{- range $current_dict := $context.Values.__daemonset_list }} + + {{- $context_novalues := omit $context "Values" }} + {{- $merged_dict := mergeOverwrite $context_novalues $current_dict.nodeData }} + {{- $_ := set $current_dict "nodeData" $merged_dict }} + {{/* Deep copy original daemonset_yaml */}} + {{- $daemonset_yaml := list $daemonset $configmap_name $serviceAccountName $merged_dict | include $daemonSetTemplateName | toString | fromYaml }} + {{- $_ := set $context.Values "__daemonset_yaml" ($daemonset_yaml | toYaml | fromYaml) }} + + {{/* Use the following name format $daemonset_root_name + sha256summ($current_dict.matchExpressions) + as labels might be too long and contain wrong characters like / */}} + {{- $_ := set $current_dict "dns_1123_name" dict }} + {{- $name_format := "" }} + {{- if eq $current_dict.name "default" }} + {{- $name_format = (printf "%s-%s" $daemonset_root_name "default") | replace "_" "-" }} + {{- else }} + {{- $name_format = (printf "%s-%s" $daemonset_root_name ($current_dict.matchExpressions | quote | sha256sum | trunc 16)) | replace "_" "-" }} + {{- end }} + {{- $_ := set $current_dict "dns_1123_name" $name_format }} + + {{/* set daemonset metadata name */}} + {{- if not $context.Values.__daemonset_yaml.metadata }}{{- $_ := set $context.Values.__daemonset_yaml "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.metadata.name }}{{- $_ := set $context.Values.__daemonset_yaml.metadata "name" dict }}{{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.metadata "name" $current_dict.dns_1123_name }} + + {{/* cross-reference configmap name to container volume definitions */}} + {{- $_ := set $context.Values "__volume_list" list }} + {{- range $current_volume := $context.Values.__daemonset_yaml.spec.template.spec.volumes }} + {{- $_ := set $context.Values "__volume" $current_volume }} + {{- if hasKey $context.Values.__volume "secret" }} + {{- if eq $context.Values.__volume.secret.secretName $configmap_name }} + {{- $_ := set $context.Values.__volume.secret "secretName" (printf "%s-etc" $current_dict.dns_1123_name) }} + {{- end }} + {{- end }} + {{- if hasKey $context.Values.__volume "configMap" }} + {{- if eq $context.Values.__volume.configMap.name $configbin_name }} + {{- $_ := set $context.Values.__volume.configMap "name" (printf "%s-bin" $current_dict.dns_1123_name) }} + {{- end }} + {{- end }} + {{- $updated_list := append $context.Values.__volume_list $context.Values.__volume }} + {{- $_ := set $context.Values "__volume_list" $updated_list }} + {{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "volumes" $context.Values.__volume_list }} + + + {{/* populate scheduling restrictions */}} + {{- if hasKey $current_dict "matchExpressions" }} + {{- $length := len $current_dict.matchExpressions }} + {{- if gt $length 0 }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec "affinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity "nodeAffinity" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity "requiredDuringSchedulingIgnoredDuringExecution" dict }}{{- end }} + + {{- $expressions_modified := list }} + {{- if hasKey $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" }} + {{- range $orig_expression := $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution.nodeSelectorTerms }} + {{- $match_expressions_modified := list }} + {{- $match_expressions_modified = concat $match_expressions_modified $current_dict.matchExpressions }} + {{- if hasKey $orig_expression "matchExpressions" }} + {{- $match_expressions_modified = concat $match_expressions_modified $orig_expression.matchExpressions }} + {{- $expressions_modified = append $expressions_modified (dict "matchExpressions" $match_expressions_modified) }} + {{- end }} + {{- end }} + {{- else }} + {{- $expressions_modified = (list (dict "matchExpressions" $current_dict.matchExpressions)) }} + {{- end }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.spec.affinity.nodeAffinity.requiredDuringSchedulingIgnoredDuringExecution "nodeSelectorTerms" $expressions_modified }} + {{- end }} + {{- end }} + + {{/* input value hash for current set of values overrides */}} + {{- if not $context.Values.__daemonset_yaml.spec }}{{- $_ := set $context.Values.__daemonset_yaml "spec" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template }}{{- $_ := set $context.Values.__daemonset_yaml.spec "template" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template "metadata" dict }}{{- end }} + {{- if not $context.Values.__daemonset_yaml.spec.template.metadata.annotations }}{{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata "annotations" dict }}{{- end }} + {{- $cmap := list (printf "%s-etc" $current_dict.dns_1123_name) $current_dict.nodeData | include $configmap_include }} + {{- $cmap_bin := list (printf "%s-bin" $current_dict.dns_1123_name) $current_dict.nodeData | include $configbin_include }} + {{- $values_cmap_hash := $cmap | quote | sha256sum }} + {{- $values_cmap_bin_hash := $cmap_bin | quote | sha256sum }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-etc-hash" $values_cmap_hash }} + {{- $_ := set $context.Values.__daemonset_yaml.spec.template.metadata.annotations "configmap-bin-hash" $values_cmap_bin_hash }} + + {{/* Do not set override for default daemonset */}} + {{- if $current_dict.daemonset_override }} + {{- $_ := set $context.Values.__daemonset_yaml.metadata.annotations "daemonset_override" $current_dict.daemonset_override }} + {{- end }} + +{{/* generate configmap */}} +--- +{{ $cmap }} + {{/* generate -bin yaml */}} +--- +{{ $cmap_bin }} + {{/* generate daemonset yaml */}} +--- +{{ $context.Values.__daemonset_yaml | toYaml }} + {{- end }} +{{- end }} diff --git a/helm-toolkit/templates/utils/_dependency_resolver.tpl b/helm-toolkit/templates/utils/_dependency_resolver.tpl new file mode 100644 index 0000000000..4a88dd8dfb --- /dev/null +++ b/helm-toolkit/templates/utils/_dependency_resolver.tpl @@ -0,0 +1,40 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "helm-toolkit.utils.dependency_resolver" }} +{{- $envAll := index . "envAll" -}} +{{- $dependencyMixinParam := index . "dependencyMixinParam" -}} +{{- $dependencyKey := index . "dependencyKey" -}} +{{- if $dependencyMixinParam -}} +{{- $_ := set $envAll.Values "pod_dependency" dict -}} +{{- if kindIs "string" $dependencyMixinParam }} +{{- if ( index $envAll.Values.dependencies.dynamic.targeted $dependencyMixinParam ) }} +{{- $_ := include "helm-toolkit.utils.merge" (tuple $envAll.Values.pod_dependency ( index $envAll.Values.dependencies.static $dependencyKey ) ( index $envAll.Values.dependencies.dynamic.targeted $dependencyMixinParam $dependencyKey ) ) -}} +{{- else }} +{{- $_ := set $envAll.Values "pod_dependency" ( index $envAll.Values.dependencies.static $dependencyKey ) }} +{{- end }} +{{- else if kindIs "slice" $dependencyMixinParam }} +{{- $_ := set $envAll.Values "__deps" ( index $envAll.Values.dependencies.static $dependencyKey ) }} +{{- range $k, $v := $dependencyMixinParam -}} +{{- if ( index $envAll.Values.dependencies.dynamic.targeted $v ) }} +{{- $_ := include "helm-toolkit.utils.merge" (tuple $envAll.Values.pod_dependency $envAll.Values.__deps ( index $envAll.Values.dependencies.dynamic.targeted $v $dependencyKey ) ) -}} +{{- $_ := set $envAll.Values "__deps" $envAll.Values.pod_dependency -}} +{{- end }} +{{- end }} +{{- end }} +{{- else -}} +{{- $_ := set $envAll.Values "pod_dependency" ( index $envAll.Values.dependencies.static $dependencyKey ) -}} +{{- end -}} +{{ $envAll.Values.pod_dependency | toYaml }} +{{- end }} diff --git a/helm-toolkit/templates/utils/_hash.tpl b/helm-toolkit/templates/utils/_hash.tpl new file mode 100644 index 0000000000..d871b62672 --- /dev/null +++ b/helm-toolkit/templates/utils/_hash.tpl @@ -0,0 +1,21 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "helm-toolkit.utils.hash" -}} +{{- $name := index . 0 -}} +{{- $context := index . 1 -}} +{{- $last := base $context.Template.Name }} +{{- $wtf := $context.Template.Name | replace $last $name -}} +{{- include $wtf $context | sha256sum | quote -}} +{{- end -}} diff --git a/helm-toolkit/templates/utils/_hash2.tpl b/helm-toolkit/templates/utils/_hash2.tpl new file mode 100644 index 0000000000..afaaee7e80 --- /dev/null +++ b/helm-toolkit/templates/utils/_hash2.tpl @@ -0,0 +1,21 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "helm-toolkit.utils.hash2" -}} +{{- $name := index . 0 -}} +{{- $context := index . 1 -}} +{{- $last := base $context.Template.Name }} +{{- $wtf := $context.Template.Name | replace $last $name -}} +{{- printf "%s" $wtf | quote -}} +{{- end -}} diff --git a/helm-toolkit/templates/utils/_host_list.tpl b/helm-toolkit/templates/utils/_host_list.tpl new file mode 100644 index 0000000000..0c32136a83 --- /dev/null +++ b/helm-toolkit/templates/utils/_host_list.tpl @@ -0,0 +1,44 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Returns a list of unique hosts for an endpoint, in yaml. +values: | + endpoints: + cluster_domain_suffix: cluster.local + oslo_db: + hosts: + default: mariadb + host_fqdn_override: + default: mariadb +usage: | + {{ tuple "oslo_db" "internal" . | include "helm-toolkit.utils.host_list" }} +return: | + hosts: + - mariadb + - mariadb.default +*/}} + +{{- define "helm-toolkit.utils.host_list" -}} +{{- $type := index . 0 -}} +{{- $endpoint := index . 1 -}} +{{- $context := index . 2 -}} +{{- $host_fqdn := tuple $type $endpoint $context | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }} +{{- $host_namespaced := tuple $type $endpoint $context | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" }} +{{- $host_short := tuple $type $endpoint $context | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +{{/* It is important that the FQDN host is 1st in this list, to ensure other function can use the 1st element for cert gen CN etc */}} +{{- $host_list := list $host_fqdn $host_namespaced $host_short | uniq }} +{{- dict "hosts" $host_list | toYaml }} +{{- end -}} diff --git a/helm-toolkit/templates/utils/_image_sync_list.tpl b/helm-toolkit/templates/utils/_image_sync_list.tpl new file mode 100644 index 0000000000..51923b6cb5 --- /dev/null +++ b/helm-toolkit/templates/utils/_image_sync_list.tpl @@ -0,0 +1,25 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "helm-toolkit.utils.image_sync_list" -}} +{{- $imageExcludeList := .Values.images.local_registry.exclude -}} +{{- $imageDict := .Values.images.tags -}} +{{- $local := dict "first" true -}} +{{- range $k, $v := $imageDict -}} +{{- if not $local.first -}},{{- end -}} +{{- if (not (has $k $imageExcludeList )) -}} +{{- index $imageDict $k -}} +{{- $_ := set $local "first" false -}} +{{- end -}}{{- end -}} +{{- end -}} diff --git a/helm-toolkit/templates/utils/_joinListWithComma.tpl b/helm-toolkit/templates/utils/_joinListWithComma.tpl new file mode 100644 index 0000000000..5eb5785591 --- /dev/null +++ b/helm-toolkit/templates/utils/_joinListWithComma.tpl @@ -0,0 +1,31 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Joins a list of values into a comma separated string +values: | + test: + - foo + - bar +usage: | + {{ include "helm-toolkit.utils.joinListWithComma" .Values.test }} +return: | + foo,bar +*/}} + +{{- define "helm-toolkit.utils.joinListWithComma" -}} +{{- $local := dict "first" true -}} +{{- range $k, $v := . -}}{{- if not $local.first -}},{{- end -}}{{- $v -}}{{- $_ := set $local "first" false -}}{{- end -}} +{{- end -}} diff --git a/helm-toolkit/templates/utils/_joinListWithCommaAndSingleQuotes.tpl b/helm-toolkit/templates/utils/_joinListWithCommaAndSingleQuotes.tpl new file mode 100644 index 0000000000..3bc68192d5 --- /dev/null +++ b/helm-toolkit/templates/utils/_joinListWithCommaAndSingleQuotes.tpl @@ -0,0 +1,32 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Joins a list of values into a comma seperated string with single quotes + around each value. +values: | + test: + - foo + - bar +usage: | + {{ include "helm-toolkit.utils.joinListWithCommaAndSingleQuotes" .Values.test }} +return: | + 'foo','bar' +*/}} + +{{- define "helm-toolkit.utils.joinListWithCommaAndSingleQuotes" -}} +{{- $local := dict "first" true -}} +{{- range $k, $v := . -}}{{- if not $local.first -}},{{- end -}}'{{- $v -}}'{{- $_ := set $local "first" false -}}{{- end -}} +{{- end -}} diff --git a/helm-toolkit/templates/utils/_joinListWithPrefix.tpl b/helm-toolkit/templates/utils/_joinListWithPrefix.tpl new file mode 100644 index 0000000000..40ebb15649 --- /dev/null +++ b/helm-toolkit/templates/utils/_joinListWithPrefix.tpl @@ -0,0 +1,32 @@ +{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Joins a list of prefixed values into a space separated string +values: | + test: + - foo + - bar +usage: | + {{ tuple "prefix" .Values.test | include "helm-toolkit.utils.joinListWithPrefix" }} +return: | + prefixfoo prefixbar +*/}} + +{{- define "helm-toolkit.utils.joinListWithPrefix" -}} +{{- $prefix := index . 0 -}} +{{- $local := dict "first" true -}} +{{- range $k, $v := index . 1 -}}{{- if not $local.first -}}{{- " " -}}{{- end -}}{{- $prefix -}}{{- $v -}}{{- $_ := set $local "first" false -}}{{- end -}} +{{- end -}} diff --git a/helm-toolkit/templates/utils/_joinListWithSpace.tpl b/helm-toolkit/templates/utils/_joinListWithSpace.tpl new file mode 100644 index 0000000000..59122807f1 --- /dev/null +++ b/helm-toolkit/templates/utils/_joinListWithSpace.tpl @@ -0,0 +1,31 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Joins a list of values into a space separated string +values: | + test: + - foo + - bar +usage: | + {{ include "helm-toolkit.utils.joinListWithSpace" .Values.test }} +return: | + foo bar +*/}} + +{{- define "helm-toolkit.utils.joinListWithSpace" -}} +{{- $local := dict "first" true -}} +{{- range $k, $v := . -}}{{- if not $local.first -}}{{- " " -}}{{- end -}}{{- $v -}}{{- $_ := set $local "first" false -}}{{- end -}} +{{- end -}} diff --git a/helm-toolkit/templates/utils/_merge.tpl b/helm-toolkit/templates/utils/_merge.tpl new file mode 100644 index 0000000000..ea80546645 --- /dev/null +++ b/helm-toolkit/templates/utils/_merge.tpl @@ -0,0 +1,135 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +Takes a tuple of values and merges into the first (target) one each subsequent +(source) one in order. If all values to merge are maps, then the tuple can be +passed as is and the target will be the result, otherwise pass a map with a +"values" key containing the tuple of values to merge, and the merge result will +be assigned to the "result" key of the passed map. + +When merging maps, for each key in the source, if the target does not define +that key, the source value is assigned. If both define the key, then the key +values are merged using this algorithm (recursively) and the result is assigned +to the target key. Slices are merged by appending them and removing any +duplicates, and when passing a map to this function and including a +"merge_same_named" key set to true, then map items from the slices with the same +value for the "name" key will be merged with each other. Any other values are +merged by simply keeping the source, and throwing away the target. +*/}} + +{{- define "helm-toolkit.utils.merge" -}} + {{- $local := dict -}} + {{- $_ := set $local "merge_same_named" false -}} + {{- if kindIs "map" $ -}} + {{- $_ := set $local "values" $.values -}} + {{- if hasKey $ "merge_same_named" -}} + {{- $_ := set $local "merge_same_named" $.merge_same_named -}} + {{- end -}} + {{- else -}} + {{- $_ := set $local "values" $ -}} + {{- end -}} + + {{- $target := first $local.values -}} + {{- range $item := rest $local.values -}} + {{- $call := dict "target" $target "source" . "merge_same_named" $local.merge_same_named -}} + {{- $_ := include "helm-toolkit.utils._merge" $call -}} + {{- $_ := set $local "result" $call.result -}} + {{- end -}} + + {{- if kindIs "map" $ -}} + {{- $_ := set $ "result" $local.result -}} + {{- end -}} +{{- end -}} + +{{- define "helm-toolkit.utils._merge" -}} + {{- $local := dict -}} + + {{- $_ := set $ "result" $.source -}} + + {{/* + TODO: Should we `fail` when trying to merge a collection (map or slice) with + either a different kind of collection or a scalar? + */}} + + {{- if and (kindIs "map" $.target) (kindIs "map" $.source) -}} + {{- range $key, $sourceValue := $.source -}} + {{- if not (hasKey $.target $key) -}} + {{- $_ := set $local "newTargetValue" $sourceValue -}} + {{- if kindIs "map" $sourceValue -}} + {{- $copy := dict -}} + {{- $call := dict "target" $copy "source" $sourceValue -}} + {{- $_ := include "helm-toolkit.utils._merge.shallow" $call -}} + {{- $_ := set $local "newTargetValue" $copy -}} + {{- end -}} + {{- else -}} + {{- $targetValue := index $.target $key -}} + {{- $call := dict "target" $targetValue "source" $sourceValue "merge_same_named" $.merge_same_named -}} + {{- $_ := include "helm-toolkit.utils._merge" $call -}} + {{- $_ := set $local "newTargetValue" $call.result -}} + {{- end -}} + {{- $_ := set $.target $key $local.newTargetValue -}} + {{- end -}} + {{- $_ := set $ "result" $.target -}} + {{- else if and (kindIs "slice" $.target) (kindIs "slice" $.source) -}} + {{- $call := dict "target" $.target "source" $.source -}} + {{- $_ := include "helm-toolkit.utils._merge.append_slice" $call -}} + {{- if $.merge_same_named -}} + {{- $_ := set $local "result" list -}} + {{- $_ := set $local "named_items" dict -}} + {{- range $item := $call.result -}} + {{- $_ := set $local "has_name_key" false -}} + {{- if kindIs "map" $item -}} + {{- if hasKey $item "name" -}} + {{- $_ := set $local "has_name_key" true -}} + {{- end -}} + {{- end -}} + + {{- if $local.has_name_key -}} + {{- if hasKey $local.named_items $item.name -}} + {{- $named_item := index $local.named_items $item.name -}} + {{- $call := dict "target" $named_item "source" $item "merge_same_named" $.merge_same_named -}} + {{- $_ := include "helm-toolkit.utils._merge" $call -}} + {{- else -}} + {{- $copy := dict -}} + {{- $copy_call := dict "target" $copy "source" $item -}} + {{- $_ := include "helm-toolkit.utils._merge.shallow" $copy_call -}} + {{- $_ := set $local.named_items $item.name $copy -}} + {{- $_ := set $local "result" (append $local.result $copy) -}} + {{- end -}} + {{- else -}} + {{- $_ := set $local "result" (append $local.result $item) -}} + {{- end -}} + {{- end -}} + {{- else -}} + {{- $_ := set $local "result" $call.result -}} + {{- end -}} + {{- $_ := set $ "result" (uniq $local.result) -}} + {{- end -}} +{{- end -}} + +{{- define "helm-toolkit.utils._merge.shallow" -}} + {{- range $key, $value := $.source -}} + {{- $_ := set $.target $key $value -}} + {{- end -}} +{{- end -}} + +{{- define "helm-toolkit.utils._merge.append_slice" -}} + {{- $local := dict -}} + {{- $_ := set $local "result" $.target -}} + {{- range $value := $.source -}} + {{- $_ := set $local "result" (append $local.result $value) -}} + {{- end -}} + {{- $_ := set $ "result" $local.result -}} +{{- end -}} diff --git a/helm-toolkit/templates/utils/_template.tpl b/helm-toolkit/templates/utils/_template.tpl new file mode 100644 index 0000000000..da56aa0eee --- /dev/null +++ b/helm-toolkit/templates/utils/_template.tpl @@ -0,0 +1,21 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "helm-toolkit.utils.template" -}} +{{- $name := index . 0 -}} +{{- $context := index . 1 -}} +{{- $last := base $context.Template.Name }} +{{- $wtf := $context.Template.Name | replace $last $name -}} +{{ include $wtf $context }} +{{- end -}} diff --git a/helm-toolkit/templates/utils/_to_ini.tpl b/helm-toolkit/templates/utils/_to_ini.tpl new file mode 100644 index 0000000000..a159364e7d --- /dev/null +++ b/helm-toolkit/templates/utils/_to_ini.tpl @@ -0,0 +1,51 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Returns INI formatted output from yaml input +values: | + conf: + paste: + filter:debug: + use: egg:oslo.middleware#debug + filter:request_id: + use: egg:oslo.middleware#request_id + filter:build_auth_context: + use: egg:keystone#build_auth_context +usage: | + {{ include "helm-toolkit.utils.to_ini" .Values.conf.paste }} +return: | + [filter:build_auth_context] + use = egg:keystone#build_auth_context + [filter:debug] + use = egg:oslo.middleware#debug + [filter:request_id] + use = egg:oslo.middleware#request_id +*/}} + +{{- define "helm-toolkit.utils.to_ini" -}} +{{- range $section, $values := . -}} +{{- if kindIs "map" $values -}} +[{{ $section }}] +{{range $key, $value := $values -}} +{{- if kindIs "slice" $value -}} +{{ $key }} = {{ include "helm-toolkit.utils.joinListWithComma" $value }} +{{else -}} +{{ $key }} = {{ $value }} +{{end -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/helm-toolkit/templates/utils/_to_k8s_env_secret_vars.tpl b/helm-toolkit/templates/utils/_to_k8s_env_secret_vars.tpl new file mode 100644 index 0000000000..885a86cc77 --- /dev/null +++ b/helm-toolkit/templates/utils/_to_k8s_env_secret_vars.tpl @@ -0,0 +1,46 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Returns yaml formatted to be used in k8s templates as container + env vars injected via secrets. This requires a secret- template to + be defined in the chart that can be used to house the desired secret + variables. For reference, see the fluentd chart. +values: | + test: + secrets: + foo: bar + +usage: | + {{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.test }} +return: | + - name: foo + valueFrom: + secretKeyRef: + name: "my-release-name-env-secret" + key: foo +*/}} + +{{- define "helm-toolkit.utils.to_k8s_env_secret_vars" -}} +{{- $context := index . 0 -}} +{{- $secrets := index . 1 -}} +{{ range $key, $config := $secrets -}} +- name: {{ $key }} + valueFrom: + secretKeyRef: + name: {{ printf "%s-%s" $context.Release.Name "env-secret" | quote }} + key: {{ $key }} +{{ end -}} +{{- end -}} diff --git a/helm-toolkit/templates/utils/_to_k8s_env_vars.tpl b/helm-toolkit/templates/utils/_to_k8s_env_vars.tpl new file mode 100644 index 0000000000..829dca6e08 --- /dev/null +++ b/helm-toolkit/templates/utils/_to_k8s_env_vars.tpl @@ -0,0 +1,39 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Returns key value pair formatted to be used in k8s templates as container + env vars. +values: | + test: + foo: bar +usage: | + {{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.test }} +return: | + - name: foo + value: "bar" +*/}} + +{{- define "helm-toolkit.utils.to_k8s_env_vars" -}} +{{range $key, $value := . -}} +{{- if kindIs "slice" $value -}} +- name: {{ $key }} + value: {{ include "helm-toolkit.utils.joinListWithComma" $value | quote }} +{{else -}} +- name: {{ $key }} + value: {{ $value | quote }} +{{ end -}} +{{- end -}} +{{- end -}} diff --git a/helm-toolkit/templates/utils/_to_kv_list.tpl b/helm-toolkit/templates/utils/_to_kv_list.tpl new file mode 100644 index 0000000000..91bdeb692c --- /dev/null +++ b/helm-toolkit/templates/utils/_to_kv_list.tpl @@ -0,0 +1,42 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Returns key value pair in INI format (key = value) +values: | + conf: + libvirt: + log_level: 3 +usage: | + {{ include "helm-toolkit.utils.to_kv_list" .Values.conf.libvirt }} +return: | + log_level = 3 +*/}} + +{{- define "helm-toolkit.utils.to_kv_list" -}} +{{- range $key, $value := . -}} +{{- if kindIs "slice" $value }} +{{ $key }} = {{ include "helm-toolkit.utils.joinListWithComma" $value | quote }} +{{- else if kindIs "string" $value }} +{{- if regexMatch "^[0-9]+$" $value }} +{{ $key }} = {{ $value }} +{{- else }} +{{ $key }} = {{ $value | quote }} +{{- end }} +{{- else }} +{{ $key }} = {{ $value }} +{{- end }} +{{- end -}} +{{- end -}} diff --git a/helm-toolkit/templates/utils/_to_oslo_conf.tpl b/helm-toolkit/templates/utils/_to_oslo_conf.tpl new file mode 100644 index 0000000000..622a86230e --- /dev/null +++ b/helm-toolkit/templates/utils/_to_oslo_conf.tpl @@ -0,0 +1,75 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Returns OSLO.conf formatted output from yaml input +values: | + conf: + keystone: + DEFAULT: # Keys at this level are used for section headings + max_token_size: 255 + oslo_messaging_notifications: + driver: # An example of a multistring option's syntax + type: multistring + values: + - messagingv2 + - log + oslo_messaging_notifications_stein: + driver: # An example of a csv option's syntax + type: csv + values: + - messagingv2 + - log + security_compliance: + password_expires_ignore_user_ids: + # Values in a list will be converted to a comma separated key + - "123" + - "456" +usage: | + {{ include "helm-toolkit.utils.to_oslo_conf" .Values.conf.keystone }} +return: | + [DEFAULT] + max_token_size = 255 + [oslo_messaging_notifications] + driver = messagingv2 + driver = log + [oslo_messaging_notifications_stein] + driver = messagingv2,log + [security_compliance] + password_expires_ignore_user_ids = 123,456 +*/}} + +{{- define "helm-toolkit.utils.to_oslo_conf" -}} +{{- range $section, $values := . -}} +{{- if kindIs "map" $values -}} +[{{ $section }}] +{{ range $key, $value := $values -}} +{{- if kindIs "slice" $value -}} +{{ $key }} = {{ include "helm-toolkit.utils.joinListWithComma" $value }} +{{ else if kindIs "map" $value -}} +{{- if eq $value.type "multistring" }} +{{- range $k, $multistringValue := $value.values -}} +{{ $key }} = {{ $multistringValue }} +{{ end -}} +{{ else if eq $value.type "csv" -}} +{{ $key }} = {{ include "helm-toolkit.utils.joinListWithComma" $value.values }} +{{ end -}} +{{- else -}} +{{ $key }} = {{ $value }} +{{ end -}} +{{- end -}} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/helm-toolkit/values.yaml b/helm-toolkit/values.yaml new file mode 100644 index 0000000000..681a92b69f --- /dev/null +++ b/helm-toolkit/values.yaml @@ -0,0 +1,16 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for utils. +# This is a YAML-formatted file. +# Declare name/value pairs to be passed into your templates. +# name: value diff --git a/kibana/Chart.yaml b/kibana/Chart.yaml new file mode 100644 index 0000000000..5639855866 --- /dev/null +++ b/kibana/Chart.yaml @@ -0,0 +1,29 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: v8.9.0 +description: OpenStack-Helm Kibana +name: kibana +version: 2024.2.0 +home: https://www.elastic.co/products/kibana +sources: + - https://github.com/elastic/kibana + - https://opendev.org/openstack/openstack-helm-infra +maintainers: + - name: OpenStack-Helm Authors +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit + version: ">= 0.1.0" +... diff --git a/kibana/templates/bin/_apache.sh.tpl b/kibana/templates/bin/_apache.sh.tpl new file mode 100644 index 0000000000..86a3f28b62 --- /dev/null +++ b/kibana/templates/bin/_apache.sh.tpl @@ -0,0 +1,44 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ev + +COMMAND="${@:-start}" + +function start () { + + if [ -f /etc/apache2/envvars ]; then + # Loading Apache2 ENV variables + source /etc/httpd/apache2/envvars + fi + # Apache gets grumpy about PID files pre-existing + rm -f /etc/httpd/logs/httpd.pid + + if [ -f /usr/local/apache2/conf/.htpasswd ]; then + htpasswd -b /usr/local/apache2/conf/.htpasswd "$ELASTICSEARCH_USERNAME" "$ELASTICSEARCH_PASSWORD" + else + htpasswd -cb /usr/local/apache2/conf/.htpasswd "$ELASTICSEARCH_USERNAME" "$ELASTICSEARCH_PASSWORD" + fi + + #Launch Apache on Foreground + exec httpd -DFOREGROUND +} + +function stop () { + apachectl -k graceful-stop +} + +$COMMAND diff --git a/kibana/templates/bin/_create_kibana_index_patterns.sh.tpl b/kibana/templates/bin/_create_kibana_index_patterns.sh.tpl new file mode 100644 index 0000000000..78672db7fe --- /dev/null +++ b/kibana/templates/bin/_create_kibana_index_patterns.sh.tpl @@ -0,0 +1,109 @@ +#!/bin/bash +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +set -ex +set -o noglob + +create_data_view() { + local index_name=$1 + curl -u "${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ + --max-time 30 \ + -X POST "${KIBANA_ENDPOINT}/api/data_views/data_view" \ + -H "kbn-xsrf: true" \ + -H "Content-Type: application/json" \ + -d "{ + \"data_view\": { + \"title\": \"${index_name}-*\", + \"timeFieldName\": \"@timestamp\" + } + }" +} + +data_view_exists() { + local index_name=$1 + local response=$(curl -s -u "${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ + --max-time 30 \ + -X GET "${KIBANA_ENDPOINT}/api/data_views" \ + -H "kbn-xsrf: true" \ + -H "Content-Type: application/json") + + if echo "$response" | grep -Fq "\"title\":\"${index_name}-*\""; then + return 0 + fi + return 1 +} + +set_default_data_view() { + local view_id=$1 + curl -u "${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ + --max-time 30 \ + -X POST "${KIBANA_ENDPOINT}/api/data_views/default" \ + -H "kbn-xsrf: true" \ + -H "Content-Type: application/json" \ + -d "{ + \"data_view_id\": \"${view_id}\", + \"force\": true + }" +} + +find_and_set_python() { + pythons="python3 python python2" + for p in ${pythons[@]}; do + python=$(which ${p}) + if [[ $? -eq 0 ]]; then + echo found python: ${python} + break + fi + done +} + +get_view_id() { + local index_name=$1 + local response=$(curl -s -u "${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ + --max-time 30 \ + -X GET "${KIBANA_ENDPOINT}/api/data_views" \ + -H "kbn-xsrf: true" \ + -H "Content-Type: application/json" | + $python -c "import sys,json; j=json.load(sys.stdin); t=[x['id'] for x in j['data_view'] if x['title'] == '${index_name}-*']; print(t[0] if len(t) else '')" + ) + echo $response +} + +# Create data views +{{- range $objectType, $indices := .Values.conf.create_kibana_indexes.indexes }} +{{- range $indices }} +while true; do + create_data_view "{{ . }}" + if data_view_exists "{{ . }}"; then + echo "Data view '{{ . }}-*' exists" + break + else + echo "Retrying creation of data view '{{ . }}-*' ..." + create_data_view "{{ . }}" + sleep 30 + fi +done + +{{- end }} +{{- end }} + +# Lookup default view id. The new Kibana view API requires the id +# instead of simply the name like the previous index API did. +find_and_set_python + +default_index="{{ .Values.conf.create_kibana_indexes.default_index }}" +default_index_id=$(get_view_id $default_index) + +set_default_data_view "$default_index_id" +echo "Default data view set to '${default_index}'." diff --git a/kibana/templates/bin/_flush_kibana_metadata.sh.tpl b/kibana/templates/bin/_flush_kibana_metadata.sh.tpl new file mode 100644 index 0000000000..458c6d7551 --- /dev/null +++ b/kibana/templates/bin/_flush_kibana_metadata.sh.tpl @@ -0,0 +1,19 @@ +#!/bin/bash +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +set -ex +echo "Deleting index created for metadata" + +curl ${CACERT_OPTION} -K- <<< "--user ${ELASTICSEARCH_USERNAME}:${ELASTICSEARCH_PASSWORD}" \ + -XDELETE "${ELASTICSEARCH_ENDPOINT}/.kibana*" diff --git a/kibana/templates/bin/_kibana.sh.tpl b/kibana/templates/bin/_kibana.sh.tpl new file mode 100644 index 0000000000..1172813cfe --- /dev/null +++ b/kibana/templates/bin/_kibana.sh.tpl @@ -0,0 +1,30 @@ +#!/bin/bash +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -e +COMMAND="${@:-start}" + +function start () { + exec /usr/share/kibana/bin/kibana \ + --elasticsearch.hosts="${ELASTICSEARCH_HOSTS}" \ + --elasticsearch.username="${ELASTICSEARCH_USERNAME}" \ + --elasticsearch.password="${ELASTICSEARCH_PASSWORD}" +} + +function stop () { + kill -TERM 1 +} + +$COMMAND diff --git a/kibana/templates/configmap-bin.yaml b/kibana/templates/configmap-bin.yaml new file mode 100644 index 0000000000..d7c3c11afa --- /dev/null +++ b/kibana/templates/configmap-bin.yaml @@ -0,0 +1,33 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_bin }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: kibana-bin +data: + apache.sh: | +{{ tuple "bin/_apache.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + kibana.sh: | +{{ tuple "bin/_kibana.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + create_kibana_index_patterns.sh: | +{{ tuple "bin/_create_kibana_index_patterns.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + flush_kibana_metadata.sh: | +{{ tuple "bin/_flush_kibana_metadata.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + image-repo-sync.sh: | +{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} +{{- end }} diff --git a/kibana/templates/configmap-etc.yaml b/kibana/templates/configmap-etc.yaml new file mode 100644 index 0000000000..6a9a07e911 --- /dev/null +++ b/kibana/templates/configmap-etc.yaml @@ -0,0 +1,27 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_etc }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: kibana-etc +type: Opaque +data: + kibana.yml: {{ toYaml .Values.conf.kibana | b64enc }} + # NOTE(portdirect): this must be last, to work round helm ~2.7 bug. +{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.httpd "key" "httpd.conf" "format" "Secret") | indent 2 }} +{{- end }} diff --git a/kibana/templates/deployment.yaml b/kibana/templates/deployment.yaml new file mode 100644 index 0000000000..2947eb7bd3 --- /dev/null +++ b/kibana/templates/deployment.yaml @@ -0,0 +1,177 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "kibanaProbeTemplate" }} +{{- $kibanaPort := tuple "kibana" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +{{- $esUser := .Values.endpoints.elasticsearch.auth.admin.username }} +{{- $esPass := .Values.endpoints.elasticsearch.auth.admin.password }} +{{- $authHeader := printf "%s:%s" $esUser $esPass | b64enc }} +httpGet: + path: /status + port: {{ $kibanaPort }} + httpHeaders: + - name: Authorization + value: Basic {{ $authHeader }} +{{- end }} + +{{- if .Values.manifests.deployment }} +{{- $envAll := . }} + +{{- $esUserSecret := .Values.secrets.elasticsearch.user }} +{{- $esUser := .Values.endpoints.elasticsearch.auth.admin.username }} +{{- $esPass := .Values.endpoints.elasticsearch.auth.admin.password }} +{{- $authHeader := printf "%s:%s" $esUser $esPass | b64enc }} + +{{- $esScheme := tuple "elasticsearch" "internal" "http" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" }} +{{- $esSvc := tuple "elasticsearch" "default" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }} +{{- $esHosts := printf "%s://%s" $esScheme $esSvc }} + +{{- $serviceAccountName := "kibana" }} +{{ tuple $envAll "kibana" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} + +{{- $kibanaPort := tuple "kibana" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kibana + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "kibana" "dashboard" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + replicas: {{ .Values.pod.replicas.kibana }} + selector: + matchLabels: +{{ tuple $envAll "kibana" "dashboard" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} +{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "kibana" "dashboard" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} +{{ dict "envAll" $envAll "podName" "kibana" "containerNames" (list "apache-proxy" "kibana" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "dashboard" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + affinity: +{{ tuple $envAll "kibana" "dashboard" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} + nodeSelector: + {{ .Values.labels.kibana.node_selector_key }}: {{ .Values.labels.kibana.node_selector_value | quote }} + initContainers: +{{ tuple $envAll "kibana" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: apache-proxy +{{ tuple $envAll "apache_proxy" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.apache_proxy | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "dashboard" "container" "apache_proxy" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/apache.sh + - start + ports: + - name: http + containerPort: {{ $kibanaPort }} + readinessProbe: + tcpSocket: + port: {{ $kibanaPort }} + initialDelaySeconds: 20 + periodSeconds: 30 + env: + - name: ELASTICSEARCH_USERNAME + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_USERNAME + - name: ELASTICSEARCH_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_PASSWORD + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: kibana-bin + mountPath: /tmp/apache.sh + subPath: apache.sh + readOnly: true + - name: kibana-etc + mountPath: /usr/local/apache2/conf/httpd.conf + subPath: httpd.conf + readOnly: true + - name: kibana +{{ tuple $envAll "kibana" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.kibana | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "dashboard" "container" "kibana" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/kibana.sh + - start + ports: + - name: kibana + containerPort: {{ tuple "kibana" "internal" "kibana" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +{{ dict "envAll" . "component" "kibana" "container" "kibana" "type" "liveness" "probeTemplate" (include "kibanaProbeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }} +{{ dict "envAll" . "component" "kibana" "container" "kibana" "type" "readiness" "probeTemplate" (include "kibanaProbeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }} + env: + - name: ELASTICSEARCH_HOSTS + value: {{ $esHosts }} + - name: ELASTICSEARCH_USERNAME + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_USERNAME + - name: ELASTICSEARCH_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_PASSWORD + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: pod-run + mountPath: /run + - name: kibana-bin + mountPath: /tmp/kibana.sh + subPath: kibana.sh + readOnly: true + - name: pod-etc-kibana + mountPath: /usr/share/kibana/config + - name: pod-optimize-kibana + mountPath: /usr/share/kibana/optimize + - name: kibana-etc + mountPath: /usr/share/kibana/config/kibana.yml + subPath: kibana.yml + readOnly: true +{{- dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.endpoints.elasticsearch.auth.admin.secret.tls.internal "path" "/etc/elasticsearch/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: pod-run + emptyDir: + medium: "Memory" + - name: pod-etc-kibana + emptyDir: {} + - name: pod-optimize-kibana + emptyDir: {} + - name: kibana-bin + configMap: + name: kibana-bin + defaultMode: 0555 + - name: kibana-etc + secret: + secretName: kibana-etc + defaultMode: 0444 +{{- dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.endpoints.elasticsearch.auth.admin.secret.tls.internal | include "helm-toolkit.snippets.tls_volume" | indent 8 }} +{{- end }} diff --git a/kibana/templates/ingress-kibana.yaml b/kibana/templates/ingress-kibana.yaml new file mode 100644 index 0000000000..87c1e83fab --- /dev/null +++ b/kibana/templates/ingress-kibana.yaml @@ -0,0 +1,23 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + + +{{- if and .Values.manifests.ingress .Values.network.kibana.ingress.public }} +{{- $envAll := . -}} +{{- $ingressOpts := dict "envAll" $envAll "backendService" "kibana" "backendServiceType" "kibana" "backendPort" "http" -}} +{{- if .Values.manifests.certificates -}} +{{- $_ := set $ingressOpts "certIssuer" .Values.endpoints.kibana.host_fqdn_override.default.tls.issuerRef.name -}} +{{- end -}} +{{ $ingressOpts | include "helm-toolkit.manifests.ingress" }} +{{- end }} diff --git a/kibana/templates/job-flush-kibana-metadata.yaml b/kibana/templates/job-flush-kibana-metadata.yaml new file mode 100644 index 0000000000..c657b1202b --- /dev/null +++ b/kibana/templates/job-flush-kibana-metadata.yaml @@ -0,0 +1,108 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +This hook is enabled for post-delete and pre-upgrade triggers. +The indices deleted by this hook are Kibana's meta indices + - .kibana + - .kibana_1 + - .kibana_2 + etc + +This is done to get around https://github.com/elastic/kibana/issues/58388 +which sometimes prevents Kibana deployments from upgrading successfully. +*/}} + +{{- if .Values.manifests.job_flush_kibana_metadata }} +{{- $envAll := . }} +{{- $esUserSecret := .Values.secrets.elasticsearch.user }} +{{- $serviceAccountName := "flush-kibana-metadata" }} +{{ tuple $envAll "flush_kibana_metadata" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: flush-kibana-metadata + labels: +{{ tuple $envAll "kibana" "flush_kibana_metadata" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + backoffLimit: {{ .Values.jobs.flush_kibana_metadata.backoffLimit }} + template: + metadata: + labels: +{{ tuple $envAll "kibana" "flush_kibana_metadata" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: + "helm.sh/hook": pre-install, post-delete, pre-upgrade + "helm.sh/hook-delete-policy": hook-succeeded +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} +{{ dict "envAll" $envAll "podName" "flush-kibana-metadata" "containerNames" (list "flush-kibana-metadata" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "flush_kibana_metadata" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + activeDeadlineSeconds: {{ .Values.jobs.flush_kibana_metadata.activeDeadlineSeconds }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} + initContainers: +{{ tuple $envAll "flush_kibana_metadata" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: flush-kibana-metadata +{{ tuple $envAll "flush_kibana_metadata" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.flush_kibana_metadata | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "flush_kibana_metadata" "container" "flush_kibana_metadata" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + env: + - name: ELASTICSEARCH_USERNAME + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_USERNAME + - name: ELASTICSEARCH_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_PASSWORD + - name: KIBANA_ENDPOINT + value: {{ tuple "kibana" "internal" "http" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} + - name: ELASTICSEARCH_ENDPOINT + value: {{ printf "%s://%s" (tuple "elasticsearch" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup") (tuple "elasticsearch" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup") }} +{{- if .Values.manifests.certificates }} + - name: CACERT_OPTION + value: "--cacert /etc/elasticsearch/certs/ca.crt" +{{- end }} + command: + - /tmp/flush_kibana_metadata.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: pod-run + mountPath: /run + - name: kibana-bin + mountPath: /tmp/flush_kibana_metadata.sh + subPath: flush_kibana_metadata.sh + readOnly: false +{{- dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.endpoints.elasticsearch.auth.admin.secret.tls.internal "path" "/etc/elasticsearch/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: pod-run + emptyDir: + medium: "Memory" + - name: kibana-bin + configMap: + name: kibana-bin + defaultMode: 0755 +{{- dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.endpoints.elasticsearch.auth.admin.secret.tls.internal | include "helm-toolkit.snippets.tls_volume" | indent 8 }} +{{- end }} diff --git a/kibana/templates/job-image-repo-sync.yaml b/kibana/templates/job-image-repo-sync.yaml new file mode 100644 index 0000000000..d0f8afff1d --- /dev/null +++ b/kibana/templates/job-image-repo-sync.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "kibana" -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/kibana/templates/job-register-kibana-indexes.yaml b/kibana/templates/job-register-kibana-indexes.yaml new file mode 100644 index 0000000000..9e64b31f33 --- /dev/null +++ b/kibana/templates/job-register-kibana-indexes.yaml @@ -0,0 +1,86 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.job_register_kibana_indexes }} +{{- $envAll := . }} +{{- $esUserSecret := .Values.secrets.elasticsearch.user }} +{{- $serviceAccountName := "register-kibana-indexes" }} +{{ tuple $envAll "register_kibana_indexes" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: register-kibana-indexes + labels: +{{ tuple $envAll "kibana" "register_kibana_indexes" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + template: + metadata: + labels: +{{ tuple $envAll "kibana" "register_kibana_indexes" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} +{{ dict "envAll" $envAll "podName" "register-kibana-indexes" "containerNames" (list "register-kibana-indexes" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "register_kibana_indexes" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} + initContainers: +{{ tuple $envAll "register_kibana_indexes" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: register-kibana-indexes +{{ tuple $envAll "register_kibana_indexes" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.register_kibana_indexes | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "register_kibana_indexes" "container" "register_kibana_indexes" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + env: + - name: ELASTICSEARCH_USERNAME + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_USERNAME + - name: ELASTICSEARCH_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $esUserSecret }} + key: ELASTICSEARCH_PASSWORD + - name: KIBANA_ENDPOINT + value: {{ tuple "kibana" "internal" "http" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} + - name: ELASTICSEARCH_ENDPOINT + value: {{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} + command: + - /tmp/create_kibana_index_patterns.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: pod-run + mountPath: /run + - name: kibana-bin + mountPath: /tmp/create_kibana_index_patterns.sh + subPath: create_kibana_index_patterns.sh + readOnly: false + volumes: + - name: pod-tmp + emptyDir: {} + - name: pod-run + emptyDir: + medium: "Memory" + - name: kibana-bin + configMap: + name: kibana-bin + defaultMode: 0755 +{{- end }} diff --git a/kibana/templates/network_policy.yaml b/kibana/templates/network_policy.yaml new file mode 100644 index 0000000000..92cbe2b1cd --- /dev/null +++ b/kibana/templates/network_policy.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.network_policy -}} +{{- $netpol_opts := dict "envAll" . "name" "application" "label" "kibana" -}} +{{ $netpol_opts | include "helm-toolkit.manifests.kubernetes_network_policy" }} +{{- end -}} diff --git a/kibana/templates/secret-elasticsearch-creds.yaml b/kibana/templates/secret-elasticsearch-creds.yaml new file mode 100644 index 0000000000..a8be9c7e7c --- /dev/null +++ b/kibana/templates/secret-elasticsearch-creds.yaml @@ -0,0 +1,29 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_elasticsearch }} +{{- $envAll := . }} +{{- $secretName := index $envAll.Values.secrets.elasticsearch.user }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} +type: Opaque +data: + ELASTICSEARCH_USERNAME: {{ .Values.endpoints.elasticsearch.auth.admin.username | b64enc }} + ELASTICSEARCH_PASSWORD: {{ .Values.endpoints.elasticsearch.auth.admin.password | b64enc }} + BIND_DN: {{ .Values.endpoints.ldap.auth.admin.bind | b64enc }} + BIND_PASSWORD: {{ .Values.endpoints.ldap.auth.admin.password | b64enc }} +{{- end }} diff --git a/kibana/templates/secret-ingress-tls.yaml b/kibana/templates/secret-ingress-tls.yaml new file mode 100644 index 0000000000..2f63ba566f --- /dev/null +++ b/kibana/templates/secret-ingress-tls.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_ingress_tls }} +{{- include "helm-toolkit.manifests.secret_ingress_tls" ( dict "envAll" . "backendServiceType" "kibana" "backendService" "kibana" ) }} +{{- end }} diff --git a/kibana/templates/secret-registry.yaml b/kibana/templates/secret-registry.yaml new file mode 100644 index 0000000000..da979b3223 --- /dev/null +++ b/kibana/templates/secret-registry.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_registry .Values.endpoints.oci_image_registry.auth.enabled }} +{{ include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) }} +{{- end }} diff --git a/kibana/templates/service-ingress-kibana.yaml b/kibana/templates/service-ingress-kibana.yaml new file mode 100644 index 0000000000..e5c149a460 --- /dev/null +++ b/kibana/templates/service-ingress-kibana.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.service_ingress .Values.network.kibana.ingress.public }} +{{- $serviceIngressOpts := dict "envAll" . "backendServiceType" "kibana" -}} +{{ $serviceIngressOpts | include "helm-toolkit.manifests.service_ingress" }} +{{- end }} diff --git a/kibana/templates/service.yaml b/kibana/templates/service.yaml new file mode 100644 index 0000000000..193427649a --- /dev/null +++ b/kibana/templates/service.yaml @@ -0,0 +1,32 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "kibana" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +spec: + ports: + - name: http + port: {{ tuple "kibana" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + {{ if .Values.network.kibana.node_port.enabled }} + nodePort: {{ .Values.network.kibana.node_port.port }} + {{ end }} + selector: +{{ tuple $envAll "kibana" "dashboard" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + {{ if .Values.network.kibana.node_port.enabled }} + type: NodePort + {{ end }} diff --git a/kibana/values.yaml b/kibana/values.yaml new file mode 100644 index 0000000000..baac6575a2 --- /dev/null +++ b/kibana/values.yaml @@ -0,0 +1,436 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +labels: + kibana: + node_selector_key: openstack-control-plane + node_selector_value: enabled + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +images: + tags: + apache_proxy: docker.io/library/httpd:2.4 + kibana: docker.elastic.co/kibana/kibana:8.9.0 + dep_check: quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal + image_repo_sync: docker.io/library/docker:17.07.0 + register_kibana_indexes: docker.io/openstackhelm/heat:wallaby-ubuntu_focal + flush_kibana_metadata: docker.io/openstackhelm/heat:wallaby-ubuntu_focal + pull_policy: IfNotPresent + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +pod: + security_context: + dashboard: + pod: + runAsUser: 1000 + container: + apache_proxy: + runAsUser: 0 + readOnlyRootFilesystem: false + kibana: + runAsNonRoot: true + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false + register_kibana_indexes: + pod: + runAsUser: 1000 + container: + register_kibana_indexes: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + flush_kibana_metadata: + pod: + runAsUser: 1000 + container: + flush_kibana_metadata: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + weight: + default: 10 + lifecycle: + upgrades: + deployments: + pod_replacement_strategy: RollingUpdate + revision_history: 3 + rolling_update: + max_surge: 3 + max_unavailable: 1 + replicas: + kibana: 1 + resources: + enabled: false + apache_proxy: + limits: + memory: "1024Mi" + cpu: "2000m" + requests: + memory: "128Mi" + cpu: "100m" + kibana: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + jobs: + image_repo_sync: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + register_kibana_indexes: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + flush_kibana_metadata: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + probes: + kibana: + kibana: + liveness: + enabled: true + params: + initialDelaySeconds: 180 + periodSeconds: 60 + readiness: + enabled: true + params: + initialDelaySeconds: 20 + periodSeconds: 30 +network_policy: + kibana: + ingress: + - {} + egress: + - {} + +secrets: + elasticsearch: + user: kibana-elasticsearch-user + oci_image_registry: + kibana: kibana-oci-image-registry-key + tls: + kibana: + kibana: + public: kibana-tls-public + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - kibana-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + kibana: + jobs: + - flush-kibana-metadata + services: + - endpoint: internal + service: elasticsearch + register_kibana_indexes: + jobs: + - flush-kibana-metadata + services: + - endpoint: internal + service: kibana + flush_kibana_metadata: + services: + - endpoint: internal + service: elasticsearch + +jobs: + flush_kibana_metadata: + backoffLimit: 6 + activeDeadlineSeconds: 600 + +conf: + httpd: | + ServerRoot "/usr/local/apache2" + + Listen 80 + + LoadModule mpm_event_module modules/mod_mpm_event.so + LoadModule authn_file_module modules/mod_authn_file.so + LoadModule authn_core_module modules/mod_authn_core.so + LoadModule authz_host_module modules/mod_authz_host.so + LoadModule authz_groupfile_module modules/mod_authz_groupfile.so + LoadModule authz_user_module modules/mod_authz_user.so + LoadModule authz_core_module modules/mod_authz_core.so + LoadModule access_compat_module modules/mod_access_compat.so + LoadModule auth_basic_module modules/mod_auth_basic.so + LoadModule ldap_module modules/mod_ldap.so + LoadModule authnz_ldap_module modules/mod_authnz_ldap.so + LoadModule reqtimeout_module modules/mod_reqtimeout.so + LoadModule filter_module modules/mod_filter.so + LoadModule proxy_html_module modules/mod_proxy_html.so + LoadModule log_config_module modules/mod_log_config.so + LoadModule env_module modules/mod_env.so + LoadModule headers_module modules/mod_headers.so + LoadModule setenvif_module modules/mod_setenvif.so + LoadModule version_module modules/mod_version.so + LoadModule proxy_module modules/mod_proxy.so + LoadModule proxy_connect_module modules/mod_proxy_connect.so + LoadModule proxy_http_module modules/mod_proxy_http.so + LoadModule proxy_balancer_module modules/mod_proxy_balancer.so + LoadModule remoteip_module modules/mod_remoteip.so + LoadModule slotmem_shm_module modules/mod_slotmem_shm.so + LoadModule slotmem_plain_module modules/mod_slotmem_plain.so + LoadModule unixd_module modules/mod_unixd.so + LoadModule status_module modules/mod_status.so + LoadModule autoindex_module modules/mod_autoindex.so + + + User daemon + Group daemon + + + + AllowOverride none + Require all denied + + + + Require all denied + + + ErrorLog /dev/stderr + + LogLevel warn + + + LogFormat "%a %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined + LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" proxy + LogFormat "%h %l %u %t \"%r\" %>s %b" common + + + LogFormat "%a %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combinedio + + + SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded + CustomLog /dev/stdout common + CustomLog /dev/stdout combined + CustomLog /dev/stdout proxy env=forwarded + + + + AllowOverride None + Options None + Require all granted + + + + RequestHeader unset Proxy early + + + + Include conf/extra/proxy-html.conf + + + + RemoteIPHeader X-Original-Forwarded-For + + ProxyPass http://localhost:{{ tuple "kibana" "internal" "kibana" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/ + ProxyPassReverse http://localhost:{{ tuple "kibana" "internal" "kibana" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/ + + + AuthName "Kibana" + AuthType Basic + AuthBasicProvider file ldap + AuthUserFile /usr/local/apache2/conf/.htpasswd + AuthLDAPBindDN {{ .Values.endpoints.ldap.auth.admin.bind }} + AuthLDAPBindPassword {{ .Values.endpoints.ldap.auth.admin.password }} + AuthLDAPURL {{ tuple "ldap" "default" "ldap" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | quote }} + Require valid-user + + + kibana: + elasticsearch: + pingTimeout: 1500 + requestTimeout: 30000 + shardTimeout: 0 + ops: + interval: 5000 + server: + rewriteBasePath: false + host: localhost + name: kibana + maxPayload: 1048576 + port: 5601 + ssl: + enabled: false + create_kibana_indexes: + indexes: + base: + - logstash + - journal + - kernel + application: + - openstack + default_index: logstash + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + kibana: + username: kibana + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + elasticsearch: + name: elasticsearch + namespace: null + auth: + admin: + username: admin + password: changeme + secret: + tls: + internal: elasticsearch-tls-api + hosts: + default: elasticsearch-logging + public: elasticsearch + host_fqdn_override: + default: null + path: + default: null + scheme: + default: http + port: + client: + default: 80 + kibana: + name: kibana + namespace: null + hosts: + default: kibana-dash + public: kibana + host_fqdn_override: + default: null + # NOTE(srwilkers): this chart supports TLS for fqdn over-ridden public + # endpoints using the following format: + # public: + # host: null + # tls: + # crt: null + # key: null + path: + default: null + scheme: + default: http + port: + kibana: + default: 5601 + http: + default: 80 + ldap: + hosts: + default: ldap + auth: + admin: + bind: "cn=admin,dc=cluster,dc=local" + password: password + host_fqdn_override: + default: null + path: + default: "/ou=People,dc=cluster,dc=local" + scheme: + default: ldap + port: + ldap: + default: 389 + +network: + kibana: + ingress: + public: true + classes: + namespace: "nginx" + cluster: "nginx-cluster" + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / + nginx.ingress.kubernetes.io/affinity: cookie + nginx.ingress.kubernetes.io/session-cookie-name: kube-ingress-session-kibana + nginx.ingress.kubernetes.io/session-cookie-hash: sha1 + nginx.ingress.kubernetes.io/session-cookie-expires: "600" + nginx.ingress.kubernetes.io/session-cookie-max-age: "600" + node_port: + enabled: false + port: 30905 + port: 5601 + +manifests: + configmap_bin: true + configmap_etc: true + deployment: true + ingress: true + job_image_repo_sync: true + network_policy: false + secret_elasticsearch: true + secret_ingress_tls: true + secret_registry: true + service: true + service_ingress: true + job_register_kibana_indexes: true + job_flush_kibana_metadata: true +... diff --git a/kube-dns/Chart.yaml b/kube-dns/Chart.yaml new file mode 100644 index 0000000000..5b49d1d9c1 --- /dev/null +++ b/kube-dns/Chart.yaml @@ -0,0 +1,30 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: v1.14.5 +description: OpenStack-Helm Kube-DNS +name: kube-dns +version: 2024.2.0 +home: https://github.com/coreos/flannel +icon: https://raw.githubusercontent.com/coreos/flannel/master/logos/flannel-horizontal-color.png +sources: + - https://github.com/coreos/flannel + - https://opendev.org/openstack/openstack-helm +maintainers: + - name: OpenStack-Helm Authors +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit + version: ">= 0.1.0" +... diff --git a/kube-dns/templates/configmap-bin.yaml b/kube-dns/templates/configmap-bin.yaml new file mode 100644 index 0000000000..61cbe2f8ec --- /dev/null +++ b/kube-dns/templates/configmap-bin.yaml @@ -0,0 +1,25 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_bin }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: kube-dns-bin +data: + image-repo-sync.sh: | +{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} +{{- end }} diff --git a/kube-dns/templates/configmap-kube-dns.yaml b/kube-dns/templates/configmap-kube-dns.yaml new file mode 100644 index 0000000000..ce2d3d3a41 --- /dev/null +++ b/kube-dns/templates/configmap-kube-dns.yaml @@ -0,0 +1,24 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_kube_dns }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: kube-dns + labels: + addonmanager.kubernetes.io/mode: EnsureExists +{{- end }} diff --git a/kube-dns/templates/deployment-kube-dns.yaml b/kube-dns/templates/deployment-kube-dns.yaml new file mode 100644 index 0000000000..d270005018 --- /dev/null +++ b/kube-dns/templates/deployment-kube-dns.yaml @@ -0,0 +1,201 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.deployment_kube_dns }} +{{- $envAll := . }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: + k8s-app: kube-dns +{{ tuple $envAll "kubernetes" "dns" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + name: kube-dns +spec: + replicas: 1 + selector: + matchLabels: + k8s-app: kube-dns +{{ tuple $envAll "kubernetes" "dns" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} + strategy: + rollingUpdate: + maxSurge: 10% + maxUnavailable: 0 + type: RollingUpdate + template: + metadata: + labels: + k8s-app: kube-dns +{{ tuple $envAll "kubernetes" "dns" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + spec: + affinity: + nodeAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + nodeSelectorTerms: + - matchExpressions: + - key: beta.kubernetes.io/arch + operator: In + values: + - amd64 + containers: + - name: kubedns +{{ tuple $envAll "kube_dns" | include "helm-toolkit.snippets.image" | indent 10 }} + args: + - --domain={{ .Values.networking.dnsDomain }}. + - --dns-port=10053 + - --config-dir=/kube-dns-config + - --v=2 + env: + - name: PROMETHEUS_PORT + value: "10055" + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthcheck/kubedns + port: 10054 + scheme: HTTP + initialDelaySeconds: 60 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + ports: + - containerPort: 10053 + name: dns-local + protocol: UDP + - containerPort: 10053 + name: dns-tcp-local + protocol: TCP + - containerPort: 10055 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /readiness + port: 8081 + scheme: HTTP + initialDelaySeconds: 3 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + resources: + limits: + memory: 170Mi + requests: + cpu: 100m + memory: 70Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - mountPath: /kube-dns-config + name: kube-dns-config + - name: dnsmasq +{{ tuple $envAll "kube_dns_nanny" | include "helm-toolkit.snippets.image" | indent 10 }} + args: + - -v=2 + - -logtostderr + - -configDir=/etc/k8s/dns/dnsmasq-nanny + - -restartDnsmasq=true + - -- + - -k + - --cache-size=1000 + - --log-facility=- + - --server=/{{ .Values.networking.dnsDomain }}/127.0.0.1#10053 + - --server=/in-addr.arpa/127.0.0.1#10053 + - --server=/ip6.arpa/127.0.0.1#10053 + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthcheck/dnsmasq + port: 10054 + scheme: HTTP + initialDelaySeconds: 60 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + resources: + requests: + cpu: 150m + memory: 20Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - mountPath: /etc/k8s/dns/dnsmasq-nanny + name: kube-dns-config + - name: sidecar +{{ tuple $envAll "kube_dns_sidecar" | include "helm-toolkit.snippets.image" | indent 10 }} + args: + - --v=2 + - --logtostderr + - --probe=kubedns,127.0.0.1:10053,kubernetes.default.svc.{{ .Values.networking.dnsDomain }},5,A + - --probe=dnsmasq,127.0.0.1:53,kubernetes.default.svc.{{ .Values.networking.dnsDomain }},5,A + livenessProbe: + failureThreshold: 5 + httpGet: + path: /metrics + port: 10054 + scheme: HTTP + initialDelaySeconds: 60 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + ports: + - containerPort: 10054 + name: metrics + protocol: TCP + resources: + requests: + cpu: 10m + memory: 20Mi + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + dnsPolicy: {{ .Values.pod.dns_policy }} + restartPolicy: Always + schedulerName: default-scheduler + securityContext: {} + serviceAccount: kube-dns + serviceAccountName: kube-dns + terminationGracePeriodSeconds: 30 + tolerations: + - key: CriticalAddonsOnly + operator: Exists + - effect: NoSchedule + key: node-role.kubernetes.io/master + - effect: NoSchedule + key: node-role.kubernetes.io/control-plane + volumes: + - name: pod-tmp + emptyDir: {} + - configMap: + defaultMode: 420 + name: kube-dns + optional: true + name: kube-dns-config +{{- end }} diff --git a/kube-dns/templates/job-image-repo-sync.yaml b/kube-dns/templates/job-image-repo-sync.yaml new file mode 100644 index 0000000000..32195cb12a --- /dev/null +++ b/kube-dns/templates/job-image-repo-sync.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "kube-dns" -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/kube-dns/templates/secret-registry.yaml b/kube-dns/templates/secret-registry.yaml new file mode 100644 index 0000000000..da979b3223 --- /dev/null +++ b/kube-dns/templates/secret-registry.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_registry .Values.endpoints.oci_image_registry.auth.enabled }} +{{ include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) }} +{{- end }} diff --git a/kube-dns/templates/service-kube-dns.yaml b/kube-dns/templates/service-kube-dns.yaml new file mode 100644 index 0000000000..aa74f76ef4 --- /dev/null +++ b/kube-dns/templates/service-kube-dns.yaml @@ -0,0 +1,43 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.service_kube_dns }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Service +metadata: + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: KubeDNS +{{ tuple $envAll "kubernetes" "dns" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + name: kube-dns +spec: + type: ClusterIP + clusterIP: {{ .Values.networking.dnsIP }} + sessionAffinity: None + ports: + - name: dns + port: 53 + protocol: UDP + targetPort: 53 + - name: dns-tcp + port: 53 + protocol: TCP + targetPort: 53 + selector: + k8s-app: kube-dns +{{ tuple $envAll "kubernetes" "dns" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- end }} diff --git a/kube-dns/templates/serviceaccount-kube-dns.yaml b/kube-dns/templates/serviceaccount-kube-dns.yaml new file mode 100644 index 0000000000..6c10146aaf --- /dev/null +++ b/kube-dns/templates/serviceaccount-kube-dns.yaml @@ -0,0 +1,31 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.serviceaccount_kube_dns }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kube-dns + labels: + kubernetes.io/cluster-service: "true" + addonmanager.kubernetes.io/mode: Reconcile +{{- if $envAll.Values.manifests.secret_registry }} +{{- if $envAll.Values.endpoints.oci_image_registry.auth.enabled }} +imagePullSecrets: + - name: {{ index $envAll.Values.secrets.oci_image_registry $envAll.Chart.Name }} +{{- end -}} +{{- end -}} +{{- end }} diff --git a/kube-dns/values.yaml b/kube-dns/values.yaml new file mode 100644 index 0000000000..1e2e188636 --- /dev/null +++ b/kube-dns/values.yaml @@ -0,0 +1,111 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# https://raw.githubusercontent.com/coreos/flannel/v0.8.0/Documentation/kube-flannel.yml + +--- +labels: + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +images: + tags: + kube_dns: registry.k8s.io/k8s-dns-kube-dns-amd64:1.14.5 + kube_dns_nanny: registry.k8s.io/k8s-dns-dnsmasq-nanny-amd64:1.14.5 + kube_dns_sidecar: registry.k8s.io/k8s-dns-sidecar-amd64:1.14.5 + dep_check: quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal + image_repo_sync: docker.io/library/docker:17.07.0 + pull_policy: IfNotPresent + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +pod: + dns_policy: "Default" + resources: + enabled: false + jobs: + image_repo_sync: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + +networking: + dnsDomain: cluster.local + dnsIP: 10.96.0.10 + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - kube-dns-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + kube_dns: + services: null + +secrets: + oci_image_registry: + kube-dns: kube-dns-oci-image-registry-key + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + kube-dns: + username: kube-dns + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + +manifests: + configmap_bin: true + configmap_kube_dns: true + deployment_kube_dns: true + job_image_repo_sync: true + secret_registry: true + service_kube_dns: true + serviceaccount_kube_dns: true +... diff --git a/kubernetes-keystone-webhook/Chart.yaml b/kubernetes-keystone-webhook/Chart.yaml new file mode 100644 index 0000000000..e5a77b3031 --- /dev/null +++ b/kubernetes-keystone-webhook/Chart.yaml @@ -0,0 +1,28 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: v0.2.0 +description: OpenStack-Helm Kubernetes keystone webhook +name: kubernetes-keystone-webhook +version: 2024.2.0 +home: https://github.com/kubernetes/cloud-provider-openstack +sources: + - https://opendev.org/openstack/openstack-helm-infra +maintainers: + - name: OpenStack-Helm Authors +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit + version: ">= 0.1.0" +... diff --git a/kubernetes-keystone-webhook/templates/bin/_kubernetes-keystone-webhook-test.sh.tpl b/kubernetes-keystone-webhook/templates/bin/_kubernetes-keystone-webhook-test.sh.tpl new file mode 100644 index 0000000000..a2d2c54b6d --- /dev/null +++ b/kubernetes-keystone-webhook/templates/bin/_kubernetes-keystone-webhook-test.sh.tpl @@ -0,0 +1,31 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +TOKEN="$(openstack token issue -f value -c id)" +cat << EOF | curl -kvs -XPOST -d @- "${WEBHOOK_URL}" | python -mjson.tool +{ + "apiVersion": "authentication.k8s.io/v1beta1", + "kind": "TokenReview", + "metadata": { + "creationTimestamp": null + }, + "spec": { + "token": "$TOKEN" + } +} +EOF diff --git a/kubernetes-keystone-webhook/templates/bin/_start.sh.tpl b/kubernetes-keystone-webhook/templates/bin/_start.sh.tpl new file mode 100644 index 0000000000..05c4188fd8 --- /dev/null +++ b/kubernetes-keystone-webhook/templates/bin/_start.sh.tpl @@ -0,0 +1,23 @@ +#!/bin/sh + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -xe + +exec /bin/k8s-keystone-auth \ + --tls-cert-file /opt/kubernetes-keystone-webhook/pki/tls.crt \ + --tls-private-key-file /opt/kubernetes-keystone-webhook/pki/tls.key \ + --keystone-policy-file /etc/kubernetes-keystone-webhook/policy.json \ + --keystone-url {{ tuple "identity" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" }} diff --git a/kubernetes-keystone-webhook/templates/configmap-bin.yaml b/kubernetes-keystone-webhook/templates/configmap-bin.yaml new file mode 100644 index 0000000000..e013fef4ca --- /dev/null +++ b/kubernetes-keystone-webhook/templates/configmap-bin.yaml @@ -0,0 +1,27 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_bin }} +{{- $envAll := . -}} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: kubernetes-keystone-webhook-bin +data: + start.sh: | +{{ tuple "bin/_start.sh.tpl" $envAll | include "helm-toolkit.utils.template" | indent 4 }} + kubernetes-keystone-webhook-test.sh: | +{{ tuple "bin/_kubernetes-keystone-webhook-test.sh.tpl" $envAll | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} diff --git a/kubernetes-keystone-webhook/templates/configmap-etc.yaml b/kubernetes-keystone-webhook/templates/configmap-etc.yaml new file mode 100644 index 0000000000..7f40a14956 --- /dev/null +++ b/kubernetes-keystone-webhook/templates/configmap-etc.yaml @@ -0,0 +1,26 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_etc }} +{{- $envAll := . -}} + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: kubernetes-keystone-webhook-etc +data: + policy.json: | +{{ toPrettyJson $envAll.Values.conf.policy | indent 4 }} +{{- end }} diff --git a/kubernetes-keystone-webhook/templates/deployment.yaml b/kubernetes-keystone-webhook/templates/deployment.yaml new file mode 100644 index 0000000000..ed052b50f1 --- /dev/null +++ b/kubernetes-keystone-webhook/templates/deployment.yaml @@ -0,0 +1,95 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.deployment }} +{{- $envAll := . }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kubernetes-keystone-webhook + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "kubernetes-keystone-webhook" "api" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + replicas: {{ $envAll.Values.pod.replicas.api }} + selector: + matchLabels: +{{ tuple $envAll "kubernetes-keystone-webhook" "api" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} + template: + metadata: + labels: +{{ tuple $envAll "kubernetes-keystone-webhook" "api" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} +{{ dict "envAll" $envAll "podName" "kubernetes-keystone-webhook" "containerNames" (list "kubernetes-keystone-webhook") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "kubernetes_keystone_webhook" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + containers: + - name: kubernetes-keystone-webhook +{{ tuple $envAll "kubernetes_keystone_webhook" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "kubernetes_keystone_webhook" "container" "kubernetes_keystone_webhook" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/start.sh + readinessProbe: + tcpSocket: + port: {{ tuple "kubernetes_keystone_webhook" "internal" "api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + initialDelaySeconds: 15 + periodSeconds: 10 + ports: + - name: k8sksauth-pub + containerPort: {{ tuple "kubernetes_keystone_webhook" "internal" "api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: etc-kubernetes-keystone-webhook + mountPath: /etc/kubernetes-keystone-webhook + - name: key-kubernetes-keystone-webhook + mountPath: /opt/kubernetes-keystone-webhook/pki/tls.crt + subPath: tls.crt + readOnly: true + - name: key-kubernetes-keystone-webhook + mountPath: /opt/kubernetes-keystone-webhook/pki/tls.key + subPath: tls.key + readOnly: true + - name: kubernetes-keystone-webhook-etc + mountPath: /etc/kubernetes-keystone-webhook/policy.json + subPath: policy.json + readOnly: true + - name: kubernetes-keystone-webhook-bin + mountPath: /tmp/start.sh + subPath: start.sh + readOnly: true + volumes: + - name: pod-tmp + emptyDir: {} + - name: etc-kubernetes-keystone-webhook + emptyDir: {} + - name: key-kubernetes-keystone-webhook + secret: + secretName: {{ $envAll.Values.secrets.certificates.api }} + defaultMode: 0444 + - name: kubernetes-keystone-webhook-etc + configMap: + name: kubernetes-keystone-webhook-etc + defaultMode: 0444 + - name: kubernetes-keystone-webhook-bin + configMap: + name: kubernetes-keystone-webhook-bin + defaultMode: 0555 +{{- end }} diff --git a/kubernetes-keystone-webhook/templates/ingress.yaml b/kubernetes-keystone-webhook/templates/ingress.yaml new file mode 100644 index 0000000000..6dde038eb3 --- /dev/null +++ b/kubernetes-keystone-webhook/templates/ingress.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.ingress_webhook .Values.network.api.ingress.public }} +{{- $ingressOpts := dict "envAll" . "backendService" "api" "backendServiceType" "kubernetes_keystone_webhook" "backendPort" "k8sksauth-pub" -}} +{{ $ingressOpts | include "helm-toolkit.manifests.ingress" }} +{{- end }} diff --git a/kubernetes-keystone-webhook/templates/pod-test.yaml b/kubernetes-keystone-webhook/templates/pod-test.yaml new file mode 100644 index 0000000000..98f685555d --- /dev/null +++ b/kubernetes-keystone-webhook/templates/pod-test.yaml @@ -0,0 +1,65 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.pod_test }} +{{- $envAll := . }} + +{{- $mounts_kubernetes_keystone_webhook_tests := $envAll.Values.pod.mounts.kubernetes_keystone_webhook_tests.kubernetes_keystone_webhook_tests }} +{{- $mounts_kubernetes_keystone_webhook_tests_init := $envAll.Values.pod.mounts.kubernetes_keystone_webhook_tests.init_container }} + +{{- $serviceAccountName := print $envAll.Release.Name "-test" }} +{{ tuple $envAll "tests" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: v1 +kind: Pod +metadata: + name: "{{ $envAll.Release.Name }}-test" + annotations: + "helm.sh/hook": test-success +spec: + serviceAccountName: {{ $serviceAccountName }} + nodeSelector: + {{ $envAll.Values.labels.test.node_selector_key }}: {{ $envAll.Values.labels.test.node_selector_value | quote }} + restartPolicy: Never + initContainers: +{{ tuple $envAll "tests" $mounts_kubernetes_keystone_webhook_tests_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 4 }} + containers: + - name: {{ $envAll.Release.Name }}-kubernetes-keystone-webhook-test +{{ tuple $envAll "scripted_test" | include "helm-toolkit.snippets.image" | indent 6 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.tests | include "helm-toolkit.snippets.kubernetes_resources" | indent 6 }} + env: + - name: WEBHOOK_URL + value: {{ tuple "kubernetes_keystone_webhook" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | quote }} +{{- with $env := dict "ksUserSecret" .Values.secrets.identity.admin }} +{{- include "helm-toolkit.snippets.keystone_openrc_env_vars" $env | indent 8 }} +{{- end }} + command: + - /tmp/kubernetes-keystone-webhook-test.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: kubernetes-keystone-webhook-bin + mountPath: /tmp/kubernetes-keystone-webhook-test.sh + subPath: kubernetes-keystone-webhook-test.sh + readOnly: true +{{ if $mounts_kubernetes_keystone_webhook_tests.volumeMounts }}{{ toYaml $mounts_kubernetes_keystone_webhook_tests.volumeMounts | indent 8 }}{{ end }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: kubernetes-keystone-webhook-bin + configMap: + name: kubernetes-keystone-webhook-bin + defaultMode: 0555 +{{ if $mounts_kubernetes_keystone_webhook_tests.volumes }}{{ toYaml $mounts_kubernetes_keystone_webhook_tests.volumes | indent 4 }}{{ end }} +{{- end }} diff --git a/kubernetes-keystone-webhook/templates/secret-certificates.yaml b/kubernetes-keystone-webhook/templates/secret-certificates.yaml new file mode 100644 index 0000000000..7cd62526ff --- /dev/null +++ b/kubernetes-keystone-webhook/templates/secret-certificates.yaml @@ -0,0 +1,26 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_certificates }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $envAll.Values.secrets.certificates.api }} +type: kubernetes.io/tls +data: + tls.crt: {{ $envAll.Values.endpoints.kubernetes.auth.api.tls.crt | default "" | b64enc }} + tls.key: {{ $envAll.Values.endpoints.kubernetes.auth.api.tls.key | default "" | b64enc }} +{{- end }} diff --git a/kubernetes-keystone-webhook/templates/secret-keystone.yaml b/kubernetes-keystone-webhook/templates/secret-keystone.yaml new file mode 100644 index 0000000000..0747b96344 --- /dev/null +++ b/kubernetes-keystone-webhook/templates/secret-keystone.yaml @@ -0,0 +1,28 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_keystone }} +{{- $envAll := . }} +{{- range $key1, $userClass := tuple "admin" }} +{{- $secretName := index $envAll.Values.secrets.identity $userClass }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} +type: Opaque +data: +{{- tuple $userClass "internal" $envAll | include "helm-toolkit.snippets.keystone_secret_openrc" | indent 2 -}} +{{- end }} +{{- end }} diff --git a/kubernetes-keystone-webhook/templates/secret-registry.yaml b/kubernetes-keystone-webhook/templates/secret-registry.yaml new file mode 100644 index 0000000000..da979b3223 --- /dev/null +++ b/kubernetes-keystone-webhook/templates/secret-registry.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_registry .Values.endpoints.oci_image_registry.auth.enabled }} +{{ include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) }} +{{- end }} diff --git a/kubernetes-keystone-webhook/templates/service-ingress-api.yaml b/kubernetes-keystone-webhook/templates/service-ingress-api.yaml new file mode 100644 index 0000000000..088a43c5cc --- /dev/null +++ b/kubernetes-keystone-webhook/templates/service-ingress-api.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.service_ingress_api .Values.network.api.ingress.public }} +{{- $serviceIngressOpts := dict "envAll" . "backendService" "api" "backendServiceType" "kubernetes_keystone_webhook" -}} +{{ $serviceIngressOpts | include "helm-toolkit.manifests.service_ingress" }} +{{- end }} diff --git a/kubernetes-keystone-webhook/templates/service.yaml b/kubernetes-keystone-webhook/templates/service.yaml new file mode 100644 index 0000000000..8e58d3974d --- /dev/null +++ b/kubernetes-keystone-webhook/templates/service.yaml @@ -0,0 +1,28 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.service }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "kubernetes_keystone_webhook" "internal" $envAll | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +spec: + ports: + - name: k8sksauth-pub + port: {{ tuple "kubernetes_keystone_webhook" "internal" "api" $envAll | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + selector: +{{ tuple $envAll "kubernetes-keystone-webhook" "api" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- end }} diff --git a/kubernetes-keystone-webhook/values.yaml b/kubernetes-keystone-webhook/values.yaml new file mode 100644 index 0000000000..65c082a51f --- /dev/null +++ b/kubernetes-keystone-webhook/values.yaml @@ -0,0 +1,575 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +labels: + api: + node_selector_key: openstack-control-plane + node_selector_value: enabled + test: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +images: + tags: + kubernetes_keystone_webhook: docker.io/k8scloudprovider/k8s-keystone-auth:v1.19.0 + scripted_test: docker.io/openstackhelm/heat:wallaby-ubuntu_focal + dep_check: quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal + image_repo_sync: docker.io/library/docker:17.07.0 + pull_policy: IfNotPresent + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +network: + api: + ingress: + public: true + classes: + namespace: "nginx" + cluster: "nginx-cluster" + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / + nginx.ingress.kubernetes.io/secure-backends: "true" + external_policy_local: false + node_port: + enabled: false + port: 30601 + +pod: + security_context: + kubernetes_keystone_webhook: + pod: + runAsUser: 65534 + container: + kubernetes_keystone_webhook: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + mandatory_access_control: + type: apparmor + kubernetes-keystone-webhook: + kubernetes-keystone-webhook: runtime/default + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + weight: + default: 10 + replicas: + api: 1 + resources: + enabled: false + api: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "256Mi" + cpu: "200m" + jobs: + tests: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "256Mi" + cpu: "200m" + mounts: + kubernetes_keystone_webhook_api: + init_container: null + kubernetes_keystone_webhook_api: null + kubernetes_keystone_webhook_tests: + init_container: null + kubernetes_keystone_webhook_tests: null + +release_group: null + +conf: + policy: + - resource: + verbs: + - "*" + resources: + - "*" + namespace: "*" + version: "*" + match: + - type: role + values: + - admin + - resource: + verbs: + - "*" + resources: + - "*" + namespace: "kube-system" + version: "*" + match: + - type: role + values: + - kube-system-admin + - resource: + verbs: + - get + - list + - watch + resources: + - "*" + namespace: "kube-system" + version: "*" + match: + - type: role + values: + - kube-system-viewer + - resource: + verbs: + - "*" + resources: + - "*" + namespace: "openstack" + version: "*" + match: + - type: project + values: + - openstack-system + - resource: + verbs: + - "*" + resources: + - "*" + namespace: "*" + version: "*" + match: + - type: role + values: + - admin_k8cluster + - nonresource: + verbs: + - "*" + path: "*" + match: + - type: role + values: + - admin_k8cluster + - resource: + resources: + - pods + - pods/attach + - pods/exec + - pods/portforward + - pods/proxy + - configmaps + - endpoints + - persistentvolumeclaims + - replicationcontrollers + - replicationcontrollers/scale + - secrets + - serviceaccounts + - services + - services/proxy + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch + namespace: "*" + version: "" + match: + - type: role + values: + - admin_k8cluster_editor + - resource: + resources: + - bindings + - events + - limitranges + - namespaces/status + - pods/log + - pods/status + - replicationcontrollers/status + - resourcequotas + - resourcequotas/status + - namespaces + verbs: + - get + - list + - watch + namespace: "*" + version: "" + match: + - type: role + values: + - admin_k8cluster_editor + - resource: + resources: + - serviceaccounts + verbs: + - impersonate + namespace: "*" + version: "" + match: + - type: role + values: + - admin_k8cluster_editor + - resource: + resources: + - daemonsets + - deployments + - deployments/rollback + - deployments/scale + - replicasets + - replicasets/scale + - statefulsets + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch + namespace: "*" + version: "apps" + match: + - type: role + values: + - admin_k8cluster_editor + - resource: + resources: + - horizontalpodautoscalers + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch + namespace: "*" + version: "autoscaling" + match: + - type: role + values: + - admin_k8cluster_editor + - resource: + resources: + - cronjobs + - jobs + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch + namespace: "*" + version: "batch" + match: + - type: role + values: + - admin_k8cluster_editor + - resource: + resources: + - daemonsets + - deployments + - deployments/rollback + - deployments/scale + - ingresses + - networkpolicies + - replicasets + - replicasets/scale + - replicationcontrollers/scale + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch + namespace: "*" + version: "extensions" + match: + - type: role + values: + - admin_k8cluster_editor + - resource: + resources: + - poddisruptionbudgets + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch + namespace: "*" + version: "policy" + match: + - type: role + values: + - admin_k8cluster_editor + - resource: + resources: + - networkpolicies + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - update + - watch + namespace: "*" + version: "networking.k8s.io" + match: + - type: role + values: + - admin_k8cluster_editor + - resource: + resources: + - configmaps + - endpoints + - persistentvolumeclaims + - pods + - replicationcontrollers + - replicationcontrollers/scale + - serviceaccounts + - services + - bindings + - events + - limitranges + - namespaces/status + - pods/log + - pods/status + - replicationcontrollers/status + - resourcequotas + - resourcequotas/status + - namespaces + verbs: + - get + - list + - watch + namespace: "*" + version: "" + match: + - type: role + values: + - admin_k8cluster_viewer + - resource: + resources: + - daemonsets + - deployments + - deployments/scale + - replicasets + - replicasets/scale + - statefulsets + verbs: + - get + - list + - watch + namespace: "*" + version: "apps" + match: + - type: role + values: + - admin_k8cluster_viewer + - resource: + resources: + - horizontalpodautoscalers + verbs: + - get + - list + - watch + namespace: "*" + version: "autoscaling" + match: + - type: role + values: + - admin_k8cluster_viewer + - resource: + resources: + - cronjobs + - jobs + verbs: + - get + - list + - watch + namespace: "*" + version: "batch" + match: + - type: role + values: + - admin_k8cluster_viewer + - resource: + resources: + - daemonsets + - deployments + - deployments/scale + - ingresses + - networkpolicies + - replicasets + - replicasets/scale + - replicationcontrollers/scale + verbs: + - get + - list + - watch + namespace: "*" + version: "extensions" + match: + - type: role + values: + - admin_k8cluster_viewer + - resource: + resources: + - poddisruptionbudgets + verbs: + - get + - list + - watch + namespace: "*" + version: "policy" + match: + - type: role + values: + - admin_k8cluster_viewer + - resource: + resources: + - networkpolicies + verbs: + - get + - list + - watch + namespace: "*" + version: "networking.k8s.io" + match: + - type: role + values: + - admin_k8cluster_viewer + +secrets: + identity: + admin: kubernetes-keystone-webhook-admin + certificates: + api: kubernetes-keystone-webhook-certs + oci_image_registry: + kubernetes-keystone-webhook: kubernetes-keystone-webhook-oci-image-registry-key + +endpoints: + cluster_domain_suffix: cluster.local + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + kubernetes-keystone-webhook: + username: kubernetes-keystone-webhook + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + kubernetes: + auth: + api: + tls: + crt: null + key: null + identity: + name: keystone + namespace: null + auth: + admin: + region_name: RegionOne + username: admin + password: password + project_name: admin + user_domain_name: default + project_domain_name: default + hosts: + default: keystone + internal: keystone-api + host_fqdn_override: + default: null + path: + default: /v3 + scheme: + default: http + port: + api: + default: 80 + internal: 5000 + kubernetes_keystone_webhook: + namespace: null + name: k8sksauth + hosts: + default: k8sksauth-api + public: k8sksauth + host_fqdn_override: + default: null + path: + default: /webhook + scheme: + default: https + port: + api: + default: 8443 + public: 443 + + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - k8sksauth-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + api: + jobs: null + services: null + +manifests: + api_secret: true + configmap_etc: true + configmap_bin: true + deployment: true + ingress_webhook: true + pod_test: true + secret_certificates: true + secret_keystone: true + secret_registry: true + service_ingress_api: true + service: true +... diff --git a/kubernetes-node-problem-detector/Chart.yaml b/kubernetes-node-problem-detector/Chart.yaml new file mode 100644 index 0000000000..53b942f0bf --- /dev/null +++ b/kubernetes-node-problem-detector/Chart.yaml @@ -0,0 +1,29 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: v1.0.0 +description: OpenStack-Helm Kubernetes Node Problem Detector +name: kubernetes-node-problem-detector +version: 2024.2.0 +home: https://github.com/kubernetes/node-problem-detector +sources: + - https://github.com/kubernetes/node-problem-detector + - https://opendev.org/openstack/openstack-helm-infra +maintainers: + - name: OpenStack-Helm Authors +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit + version: ">= 0.1.0" +... diff --git a/kubernetes-node-problem-detector/templates/bin/_node-problem-detector.sh.tpl b/kubernetes-node-problem-detector/templates/bin/_node-problem-detector.sh.tpl new file mode 100644 index 0000000000..d0e4e27bcb --- /dev/null +++ b/kubernetes-node-problem-detector/templates/bin/_node-problem-detector.sh.tpl @@ -0,0 +1,25 @@ +#!/bin/sh +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +exec /opt/node-problem-detector/bin/node-problem-detector \ + {{- range $monitor, $monitorConfig := .Values.conf.monitors }} + {{- if $monitorConfig.enabled }} + --config.{{$monitor}}={{ include "helm-toolkit.utils.joinListWithComma" $monitorConfig.enabled }} \ + {{- end }} + {{- end }} + --logtostderr \ + --prometheus-address=0.0.0.0 diff --git a/kubernetes-node-problem-detector/templates/configmap-bin.yaml b/kubernetes-node-problem-detector/templates/configmap-bin.yaml new file mode 100644 index 0000000000..83531d1a4c --- /dev/null +++ b/kubernetes-node-problem-detector/templates/configmap-bin.yaml @@ -0,0 +1,36 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_bin }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: node-problem-detector-bin +data: + node-problem-detector.sh: | +{{ tuple "bin/_node-problem-detector.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + image-repo-sync.sh: | +{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} +{{- range $monitor, $monitorConfig := $envAll.Values.conf.monitors }} +{{- $scripts := $monitorConfig.scripts }} +{{- range $script, $scriptSource := $scripts.source }} +{{- if has $script $scripts.enabled }} + {{$script}}: | +{{$scriptSource | indent 4 -}} +{{- end }} +{{- end -}} +{{- end -}} +{{- end }} diff --git a/kubernetes-node-problem-detector/templates/configmap-etc.yaml b/kubernetes-node-problem-detector/templates/configmap-etc.yaml new file mode 100644 index 0000000000..1afae8faf1 --- /dev/null +++ b/kubernetes-node-problem-detector/templates/configmap-etc.yaml @@ -0,0 +1,31 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_etc }} + +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: node-problem-detector-etc +type: Opaque +data: +{{- range $monitor, $monitorConfig := $envAll.Values.conf.monitors }} +{{- $plugins := $monitorConfig.config }} +{{- range $plugin, $config := $plugins }} + {{$plugin}}.json: {{ toJson $config | b64enc }} +{{- end }} +{{ end }} +{{- end }} diff --git a/kubernetes-node-problem-detector/templates/daemonset.yaml b/kubernetes-node-problem-detector/templates/daemonset.yaml new file mode 100644 index 0000000000..7d93e3da1b --- /dev/null +++ b/kubernetes-node-problem-detector/templates/daemonset.yaml @@ -0,0 +1,135 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.daemonset }} +{{- $envAll := . }} + +{{- $serviceAccountName := printf "%s-%s" .Release.Name "node-problem-detector" }} +{{ tuple $envAll "node_problem_detector" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: run-node-problem-detector +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: cluster-admin + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: node-problem-detector + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "node_problem_detector" "metrics" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + selector: + matchLabels: +{{ tuple $envAll "node_problem_detector" "metrics" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} +{{ tuple $envAll "node_problem_detector" | include "helm-toolkit.snippets.kubernetes_upgrades_daemonset" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "node_problem_detector" "metrics" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{- if .Values.monitoring.prometheus.pod.enabled }} +{{- $prometheus_annotations := $envAll.Values.monitoring.prometheus.node_problem_detector }} +{{ tuple $prometheus_annotations | include "helm-toolkit.snippets.prometheus_pod_annotations" | indent 8 }} +{{- end }} +{{ dict "envAll" $envAll "podName" "node-problem-detector" "containerNames" (list "node-problem-detector") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + spec: +{{ dict "envAll" $envAll "application" "node_problem_detector" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} +{{ if .Values.pod.tolerations.node_problem_detector.enabled }} +{{ tuple $envAll "node_exporter" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }} +{{ else }} + nodeSelector: + {{ .Values.labels.node_problem_detector.node_selector_key }}: {{ .Values.labels.node_problem_detector.node_selector_value | quote }} +{{ end }} + containers: + - name: node-problem-detector +{{ tuple $envAll "node_problem_detector" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.node_problem_detector | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "node_problem_detector" "container" "node_problem_detector" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/node-problem-detector.sh + ports: + - name: metrics + containerPort: {{ tuple "node_problem_detector" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: log + mountPath: /var/log + readOnly: true + - name: kmsg + mountPath: /dev/kmsg + readOnly: true + - name: localtime + mountPath: /etc/localtime + readOnly: true + - name: node-problem-detector-bin + mountPath: /tmp/node-problem-detector.sh + subPath: node-problem-detector.sh + readOnly: true + {{- range $monitor, $monitorConfig := $envAll.Values.conf.monitors }} + {{- $scripts := $monitorConfig.scripts }} + {{- range $script, $scriptSource := $scripts.source }} + {{- if has $script $scripts.enabled }} + - name: node-problem-detector-bin + mountPath: /config/plugin/{{$script}} + subPath: {{$script}} + {{- end }} + {{- end }} + {{- end }} + {{- range $monitor, $monitorConfig := $envAll.Values.conf.monitors }} + {{- $plugins := $monitorConfig.config }} + {{- range $plugin, $config := $plugins }} + - name: node-problem-detector-etc + mountPath: /config/{{$plugin}}.json + subPath: {{$plugin}}.json + {{- end }} + {{- end }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: log + hostPath: + path: /var/log + - name: kmsg + hostPath: + path: /dev/kmsg + - name: localtime + hostPath: + path: /etc/localtime + - name: node-problem-detector-etc + secret: + secretName: node-problem-detector-etc + defaultMode: 292 + - name: node-problem-detector-bin + configMap: + name: node-problem-detector-bin + defaultMode: 365 +{{- end }} diff --git a/kubernetes-node-problem-detector/templates/job-image-repo-sync.yaml b/kubernetes-node-problem-detector/templates/job-image-repo-sync.yaml new file mode 100644 index 0000000000..c28a7d3798 --- /dev/null +++ b/kubernetes-node-problem-detector/templates/job-image-repo-sync.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "node-problem-detector" -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/kubernetes-node-problem-detector/templates/secret-registry.yaml b/kubernetes-node-problem-detector/templates/secret-registry.yaml new file mode 100644 index 0000000000..da979b3223 --- /dev/null +++ b/kubernetes-node-problem-detector/templates/secret-registry.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_registry .Values.endpoints.oci_image_registry.auth.enabled }} +{{ include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) }} +{{- end }} diff --git a/kubernetes-node-problem-detector/templates/service.yaml b/kubernetes-node-problem-detector/templates/service.yaml new file mode 100644 index 0000000000..ef13af4b05 --- /dev/null +++ b/kubernetes-node-problem-detector/templates/service.yaml @@ -0,0 +1,38 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.service }} +{{- $envAll := . }} +{{- $prometheus_annotations := $envAll.Values.monitoring.prometheus.node_problem_detector }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "node_problem_detector" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + labels: +{{ tuple $envAll "node_problem_detector" "metrics" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: +{{- if .Values.monitoring.prometheus.service.enabled }} +{{ tuple $prometheus_annotations | include "helm-toolkit.snippets.prometheus_service_annotations" | indent 4 }} +{{- end }} +spec: + type: ClusterIP + clusterIP: None + ports: + - name: metrics + port: {{ tuple "node_problem_detector" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + targetPort: {{ tuple "node_problem_detector" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + selector: +{{ tuple $envAll "node_problem_detector" "metrics" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- end }} diff --git a/kubernetes-node-problem-detector/values.yaml b/kubernetes-node-problem-detector/values.yaml new file mode 100644 index 0000000000..073c4a9076 --- /dev/null +++ b/kubernetes-node-problem-detector/values.yaml @@ -0,0 +1,484 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for node-exporter. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +--- +images: + tags: + node_problem_detector: docker.io/openstackhelm/node-problem-detector:latest-ubuntu_jammy + dep_check: quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal + image_repo_sync: docker.io/library/docker:17.07.0 + pull_policy: IfNotPresent + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +labels: + node_problem_detector: + node_selector_key: openstack-control-plane + node_selector_value: enabled + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +secrets: + oci_image_registry: + kubernetes-node-problem-detector: kubernetes-node-problem-detector-oci-image-registry-key + +pod: + security_context: + node_problem_detector: + pod: + runAsUser: 0 + container: + node_problem_detector: + readOnlyRootFilesystem: true + privileged: true + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + mounts: + node_problem_detector: + node_problem_detector: + init_container: null + lifecycle: + upgrades: + daemonsets: + pod_replacement_strategy: RollingUpdate + node_problem_detector: + enabled: true + min_ready_seconds: 0 + revision_history: 3 + pod_replacement_strategy: RollingUpdate + rolling_update: + max_unavailable: 1 + max_surge: 3 + termination_grace_period: + node_problem_detector: + timeout: 30 + resources: + enabled: false + node_problem_detector: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + jobs: + image_repo_sync: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + tolerations: + node_problem_detector: + enabled: false + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + - key: node-role.kubernetes.io/control-plane + operator: Exists + - key: node-role.kubernetes.io/node + operator: Exists +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - node-exporter-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + node_problem_detector: + services: null + +monitoring: + prometheus: + pod: + enabled: true + service: + enabled: false + node_problem_detector: + scrape: true + port: 20257 + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + kubernetes-node-problem-detector: + username: kubernetes-node-problem-detector + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + node_problem_detector: + name: node-problem-detector + namespace: null + hosts: + default: node-problem-detector + host_fqdn_override: + default: null + path: + default: null + port: + metrics: + default: 20257 + +manifests: + configmap_bin: true + configmap_etc: true + daemonset: true + job_image_repo_sync: true + secret_registry: true + service: false + +conf: + monitors: + system-log-monitor: + enabled: + - /config/kernel-monitor.json + - /config/docker-monitor.json + - /config/systemd-monitor.json + scripts: + enabled: null + source: null + config: + kernel-monitor: + plugin: kmsg + logPath: "/dev/kmsg" + lookback: 5m + bufferSize: 10 + source: kernel-monitor + conditions: + - type: KernelDeadlock + reason: KernelHasNoDeadlock + message: kernel has no deadlock + - type: ReadonlyFilesystem + reason: FilesystemIsNotReadOnly + message: Filesystem is not read-only + rules: + - type: temporary + reason: OOMKilling + pattern: Kill process \d+ (.+) score \d+ or sacrifice child\nKilled process \d+ + (.+) total-vm:\d+kB, anon-rss:\d+kB, file-rss:\d+kB.* + - type: temporary + reason: TaskHung + pattern: task \S+:\w+ blocked for more than \w+ seconds\. + - type: temporary + reason: UnregisterNetDevice + pattern: 'unregister_netdevice: waiting for \w+ to become free. Usage count = \d+' + - type: temporary + reason: KernelOops + pattern: 'BUG: unable to handle kernel NULL pointer dereference at .*' + - type: temporary + reason: KernelOops + pattern: 'divide error: 0000 \[#\d+\] SMP' + - type: permanent + condition: KernelDeadlock + reason: AUFSUmountHung + pattern: task umount\.aufs:\w+ blocked for more than \w+ seconds\. + - type: permanent + condition: KernelDeadlock + reason: DockerHung + pattern: task docker:\w+ blocked for more than \w+ seconds\. + - type: permanent + condition: ReadonlyFilesystem + reason: FilesystemIsReadOnly + pattern: Remounting filesystem read-only + kernel-monitor-filelog: + plugin: filelog + pluginConfig: + timestamp: "^.{15}" + message: 'kernel: \[.*\] (.*)' + timestampFormat: Jan _2 15:04:05 + logPath: "/var/log/kern.log" + lookback: 5m + bufferSize: 10 + source: kernel-monitor + conditions: + - type: KernelDeadlock + reason: KernelHasNoDeadlock + message: kernel has no deadlock + rules: + - type: temporary + reason: OOMKilling + pattern: Kill process \d+ (.+) score \d+ or sacrifice child\nKilled process \d+ + (.+) total-vm:\d+kB, anon-rss:\d+kB, file-rss:\d+kB.* + - type: temporary + reason: TaskHung + pattern: task \S+:\w+ blocked for more than \w+ seconds\. + - type: temporary + reason: UnregisterNetDevice + pattern: 'unregister_netdevice: waiting for \w+ to become free. Usage count = \d+' + - type: temporary + reason: KernelOops + pattern: 'BUG: unable to handle kernel NULL pointer dereference at .*' + - type: temporary + reason: KernelOops + pattern: 'divide error: 0000 \[#\d+\] SMP' + - type: permanent + condition: KernelDeadlock + reason: AUFSUmountHung + pattern: task umount\.aufs:\w+ blocked for more than \w+ seconds\. + - type: permanent + condition: KernelDeadlock + reason: DockerHung + pattern: task docker:\w+ blocked for more than \w+ seconds\. + kernel-monitor-counter: + plugin: custom + pluginConfig: + invoke_interval: 5m + timeout: 1m + max_output_length: 80 + concurrency: 1 + source: kernel-monitor + conditions: + - type: FrequentUnregisterNetDevice + reason: NoFrequentUnregisterNetDevice + message: node is functioning properly + rules: + - type: permanent + condition: FrequentUnregisterNetDevice + reason: UnregisterNetDevice + path: "/home/kubernetes/bin/log-counter" + args: + - "--journald-source=kernel" + - "--log-path=/var/log/journal" + - "--lookback=20m" + - "--count=3" + - "--pattern=unregister_netdevice: waiting for \\w+ to become free. Usage count + = \\d+" + timeout: 1m + docker-monitor: + plugin: journald + pluginConfig: + source: dockerd + logPath: "/var/log/journal" + lookback: 5m + bufferSize: 10 + source: docker-monitor + conditions: [] + rules: + - type: temporary + reason: CorruptDockerImage + pattern: 'Error trying v2 registry: failed to register layer: rename /var/lib/docker/image/(.+) + /var/lib/docker/image/(.+): directory not empty.*' + docker-monitor-filelog: + plugin: filelog + pluginConfig: + timestamp: ^time="(\S*)" + message: |- + msg="([^ + ]*)" + timestampFormat: '2006-01-02T15:04:05.999999999-07:00' + logPath: "/var/log/docker.log" + lookback: 5m + bufferSize: 10 + source: docker-monitor + conditions: [] + rules: + - type: temporary + reason: CorruptDockerImage + pattern: 'Error trying v2 registry: failed to register layer: rename /var/lib/docker/image/(.+) + /var/lib/docker/image/(.+): directory not empty.*' + docker-monitor-counter: + plugin: custom + pluginConfig: + invoke_interval: 5m + timeout: 1m + max_output_length: 80 + concurrency: 1 + source: docker-monitor + conditions: + - type: CorruptDockerOverlay2 + reason: NoCorruptDockerOverlay2 + message: docker overlay2 is functioning properly + rules: + - type: permanent + condition: CorruptDockerOverlay2 + reason: CorruptDockerOverlay2 + path: "/home/kubernetes/bin/log-counter" + args: + - "--journald-source=dockerd" + - "--log-path=/var/log/journal" + - "--lookback=5m" + - "--count=10" + - "--pattern=returned error: readlink /var/lib/docker/overlay2.*: invalid argument.*" + timeout: 1m + systemd-monitor: + plugin: journald + pluginConfig: + source: systemd + logPath: "/var/log/journal" + lookback: 5m + bufferSize: 10 + source: systemd-monitor + conditions: [] + rules: + - type: temporary + reason: KubeletStart + pattern: Started Kubernetes kubelet. + - type: temporary + reason: DockerStart + pattern: Starting Docker Application Container Engine... + - type: temporary + reason: ContainerdStart + pattern: Starting containerd container runtime... + systemd-monitor-counter: + plugin: custom + pluginConfig: + invoke_interval: 5m + timeout: 1m + max_output_length: 80 + concurrency: 1 + source: systemd-monitor + conditions: + - type: FrequentKubeletRestart + reason: NoFrequentKubeletRestart + message: kubelet is functioning properly + - type: FrequentDockerRestart + reason: NoFrequentDockerRestart + message: docker is functioning properly + - type: FrequentContainerdRestart + reason: NoFrequentContainerdRestart + message: containerd is functioning properly + rules: + - type: permanent + condition: FrequentKubeletRestart + reason: FrequentKubeletRestart + path: "/home/kubernetes/bin/log-counter" + args: + - "--journald-source=systemd" + - "--log-path=/var/log/journal" + - "--lookback=20m" + - "--delay=5m" + - "--count=5" + - "--pattern=Started Kubernetes kubelet." + timeout: 1m + - type: permanent + condition: FrequentDockerRestart + reason: FrequentDockerRestart + path: "/home/kubernetes/bin/log-counter" + args: + - "--journald-source=systemd" + - "--log-path=/var/log/journal" + - "--lookback=20m" + - "--count=5" + - "--pattern=Starting Docker Application Container Engine..." + timeout: 1m + - type: permanent + condition: FrequentContainerdRestart + reason: FrequentContainerdRestart + path: "/home/kubernetes/bin/log-counter" + args: + - "--journald-source=systemd" + - "--log-path=/var/log/journal" + - "--lookback=20m" + - "--count=5" + - "--pattern=Starting containerd container runtime..." + timeout: 1m + custom-plugin-monitor: + enabled: + - /config/network-problem-monitor.json + scripts: + enabled: + - network_problem.sh + source: + network_problem.sh: | + #!/bin/bash + + # This plugin checks for common network issues. Currently, it only checks + # if the conntrack table is 50% full. + set -eu + set -o pipefail + + conntrack_threshold=$(($(cat /proc/sys/net/netfilter/nf_conntrack_max)/2 )) + conntrack_count=$(cat /proc/sys/net/netfilter/nf_conntrack_count) + + if [ "$conntrack_count" -ge "$conntrack_threshold" ]; then + echo "Conntrack table approaching full" + exit 1 + fi + + exit 0 + config: + network-problem-monitor: + plugin: custom + pluginConfig: + invoke_interval: 30s + timeout: 5s + max_output_length: 80 + concurrency: 3 + source: network-custom-plugin-monitor + conditions: [] + rules: + - type: temporary + reason: ConntrackFull + path: "./config/plugin/network_problem.sh" + timeout: 3s + system-stats-monitor: + enabled: + - /config/system-stats-monitor.json + scripts: + enabled: null + source: null + config: + system-stats-monitor: + disk: + metricsConfigs: + disk/io_time: + displayName: disk/io_time + disk/weighted_io: + displayName: disk/weighted_io + disk/avg_queue_len: + displayName: disk/avg_queue_len + includeRootBlk: true + includeAllAttachedBlk: true + lsblkTimeout: 5s + invokeInterval: 60s +... diff --git a/ldap/.helmignore b/ldap/.helmignore new file mode 100644 index 0000000000..8fdbe6895d --- /dev/null +++ b/ldap/.helmignore @@ -0,0 +1,22 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.pyc +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/ldap/Chart.yaml b/ldap/Chart.yaml new file mode 100644 index 0000000000..4c8e5f3f6b --- /dev/null +++ b/ldap/Chart.yaml @@ -0,0 +1,26 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: v1.2.0 +description: OpenStack-Helm LDAP +name: ldap +version: 2024.2.0 +home: https://www.openldap.org/ +maintainers: + - name: OpenStack-Helm Authors +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit + version: ">= 0.1.0" +... diff --git a/ldap/templates/_helpers.tpl b/ldap/templates/_helpers.tpl new file mode 100644 index 0000000000..c2a40b8821 --- /dev/null +++ b/ldap/templates/_helpers.tpl @@ -0,0 +1,22 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +*/}} +{{- define "fullname" -}} +{{- $name := default .Chart.Name .Values.nameOverride -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{- define "splitdomain" -}} +{{- $name := index . 0 -}} +{{- $local := dict "first" true }} +{{- range $k, $v := splitList "." $name }}{{- if not $local.first -}},{{- end -}}dc={{- $v -}}{{- $_ := set $local "first" false -}}{{- end -}} +{{- end -}} diff --git a/ldap/templates/bin/_bootstrap.sh.tpl b/ldap/templates/bin/_bootstrap.sh.tpl new file mode 100644 index 0000000000..c29b8e7af3 --- /dev/null +++ b/ldap/templates/bin/_bootstrap.sh.tpl @@ -0,0 +1,8 @@ +#!/bin/bash +set -xe + +{{- $url := tuple "ldap" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }} +{{- $port := tuple "ldap" "internal" "ldap" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +LDAPHOST="{{ .Values.endpoints.ldap.scheme }}://{{ $url }}:{{ $port }}" +ADMIN="cn={{ .Values.secrets.identity.admin }},{{ tuple .Values.openldap.domain . | include "splitdomain" }}" +ldapadd -x -D $ADMIN -H $LDAPHOST -w {{ .Values.openldap.password }} -f /etc/sample_data.ldif diff --git a/ldap/templates/configmap-bin.yaml b/ldap/templates/configmap-bin.yaml new file mode 100644 index 0000000000..b42dbe9f29 --- /dev/null +++ b/ldap/templates/configmap-bin.yaml @@ -0,0 +1,25 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{- if .Values.manifests.configmap_bin }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: ldap-bin +data: +{{- if .Values.bootstrap.enabled }} + bootstrap.sh: | +{{ tuple "bin/_bootstrap.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} +{{- end }} diff --git a/ldap/templates/configmap-etc.yaml b/ldap/templates/configmap-etc.yaml new file mode 100644 index 0000000000..7ecbf11ac5 --- /dev/null +++ b/ldap/templates/configmap-etc.yaml @@ -0,0 +1,26 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_etc }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: ldap-etc +type: Opaque +data: +{{- if .Values.bootstrap.enabled }} + sample_data.ldif: {{ .Values.data.sample | b64enc }} +{{- end }} +{{- end }} diff --git a/ldap/templates/job-bootstrap.yaml b/ldap/templates/job-bootstrap.yaml new file mode 100644 index 0000000000..bf96682836 --- /dev/null +++ b/ldap/templates/job-bootstrap.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_bootstrap .Values.bootstrap.enabled }} +{{- $bootstrapJob := dict "envAll" . "serviceName" "ldap" "configFile" "/etc/sample_data.ldif" "keystoneUser" "admin" "openrc" "false" -}} +{{ $bootstrapJob | include "helm-toolkit.manifests.job_bootstrap" }} +{{- end }} diff --git a/ldap/templates/job-image-repo-sync.yaml b/ldap/templates/job-image-repo-sync.yaml new file mode 100644 index 0000000000..43571ea69c --- /dev/null +++ b/ldap/templates/job-image-repo-sync.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "ldap" -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/ldap/templates/network_policy.yaml b/ldap/templates/network_policy.yaml new file mode 100644 index 0000000000..5bdf6ecb18 --- /dev/null +++ b/ldap/templates/network_policy.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{- if .Values.manifests.network_policy -}} +{{- $netpol_opts := dict "envAll" . "name" "application" "label" "ldap" -}} +{{ $netpol_opts | include "helm-toolkit.manifests.kubernetes_network_policy" }} +{{- end -}} diff --git a/ldap/templates/secret-registry.yaml b/ldap/templates/secret-registry.yaml new file mode 100644 index 0000000000..da979b3223 --- /dev/null +++ b/ldap/templates/secret-registry.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_registry .Values.endpoints.oci_image_registry.auth.enabled }} +{{ include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) }} +{{- end }} diff --git a/ldap/templates/service.yaml b/ldap/templates/service.yaml new file mode 100644 index 0000000000..244f60ecc7 --- /dev/null +++ b/ldap/templates/service.yaml @@ -0,0 +1,28 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.service }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "ldap" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +spec: + ports: + - name: ldap + port: {{ tuple "ldap" "internal" "ldap" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + selector: +{{ tuple $envAll "ldap" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- end }} diff --git a/ldap/templates/statefulset.yaml b/ldap/templates/statefulset.yaml new file mode 100644 index 0000000000..848154de56 --- /dev/null +++ b/ldap/templates/statefulset.yaml @@ -0,0 +1,98 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.statefulset }} +{{- $envAll := . }} + +{{- $serviceAccountName := "ldap" }} +{{ tuple $envAll "ldap" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: ldap + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "ldap" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + serviceName: {{ tuple "ldap" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + replicas: {{ .Values.pod.replicas.server }} + selector: + matchLabels: +{{ tuple $envAll "ldap" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} + template: + metadata: + labels: +{{ tuple $envAll "ldap" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} + spec: + serviceAccountName: {{ $serviceAccountName }} + affinity: +{{ tuple $envAll "ldap" "server" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} + nodeSelector: + {{ .Values.labels.server.node_selector_key }}: {{ .Values.labels.server.node_selector_value | quote }} + initContainers: +{{ tuple $envAll "ldap" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 6 }} + containers: + - name: ldap +{{ tuple $envAll "ldap" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + env: + - name: LDAP_DOMAIN + value: {{ .Values.openldap.domain }} + - name: LDAP_ADMIN_PASSWORD + value: {{ .Values.openldap.password }} + ports: + - containerPort: {{ tuple "ldap" "internal" "ldap" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: ldap-data + mountPath: /var/lib/ldap + - name: ldap-config + mountPath: /etc/ldap/slapd.d + volumes: + - name: pod-tmp + emptyDir: {} +{{- if not .Values.storage.pvc.enabled }} + - name: ldap-data + hostPath: + path: {{ .Values.storage.host.data_path }} + - name: ldap-config + hostPath: + path: {{ .Values.storage.host.config_path }} +{{- else }} + volumeClaimTemplates: + - metadata: + name: ldap-data + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: {{ .Values.storage.pvc.class_name }} + resources: + requests: + storage: {{ .Values.storage.pvc.size }} + - metadata: + name: ldap-config + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: {{ .Values.storage.pvc.class_name }} + resources: + requests: + storage: {{ .Values.storage.pvc.size }} +{{- end }} +{{- end }} diff --git a/ldap/values.yaml b/ldap/values.yaml new file mode 100644 index 0000000000..72cae0865a --- /dev/null +++ b/ldap/values.yaml @@ -0,0 +1,264 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for ldap. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +--- +pod: + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + weight: + default: 10 + replicas: + server: 1 + lifecycle: + upgrades: + deployments: + revision_history: 3 + pod_replacement_strategy: RollingUpdate + rolling_update: + max_unavailable: 1 + max_surge: 3 + resources: + enabled: false + server: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + jobs: + bootstrap: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + image_repo_sync: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + mounts: + ldap_data_load: + init_container: null + ldap_data_load: + +images: + tags: + bootstrap: "docker.io/osixia/openldap:1.2.0" + ldap: "docker.io/osixia/openldap:1.2.0" + dep_check: quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal + image_repo_sync: docker.io/library/docker:17.07.0 + pull_policy: IfNotPresent + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - ldap-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + ldap: + jobs: null + bootstrap: + services: + - endpoint: internal + service: ldap + server: + jobs: + - ldap-load-data + services: + - endpoint: internal + service: ldap + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + +storage: + pvc: + enabled: true + size: 2Gi + class_name: general + host: + data_path: /data/openstack-helm/ldap + config_path: /data/openstack-helm/config + +labels: + server: + node_selector_key: openstack-control-plane + node_selector_value: enabled + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +bootstrap: + enabled: false + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + ldap: + username: ldap + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + ldap: + hosts: + default: ldap + host_fqdn_override: + default: null + path: null + scheme: 'ldap' + port: + ldap: + default: 389 + +network_policy: + ldap: + ingress: + - {} + egress: + - {} + +data: + sample: | + dn: ou=People,dc=cluster,dc=local + objectclass: organizationalunit + ou: People + description: We the People + + # NOTE: Password is "password" without quotes + dn: uid=alice,ou=People,dc=cluster,dc=local + objectClass: inetOrgPerson + objectClass: top + objectClass: posixAccount + objectClass: shadowAccount + objectClass: person + sn: Alice + cn: alice + uid: alice + userPassword: {SSHA}+i3t/DLCgLDGaIOAmfeFJ2kDeJWmPUDH + description: SHA + gidNumber: 1000 + uidNumber: 1493 + homeDirectory: /home/alice + mail: alice@example.com + + # NOTE: Password is "password" without quotes + dn: uid=bob,ou=People,dc=cluster,dc=local + objectClass: inetOrgPerson + objectClass: top + objectClass: posixAccount + objectClass: shadowAccount + objectClass: person + sn: Bob + cn: bob + uid: bob + userPassword: {SSHA}fCJ5vuW1BQ4/OfOVkkx1qjwi7yHFuGNB + description: MD5 + gidNumber: 1000 + uidNumber: 5689 + homeDirectory: /home/bob + mail: bob@example.com + + dn: ou=Groups,dc=cluster,dc=local + objectclass: organizationalunit + ou: Groups + description: We the People + + dn: cn=cryptography,ou=Groups,dc=cluster,dc=local + objectclass: top + objectclass: posixGroup + gidNumber: 418 + cn: cryptography + description: Cryptography Team + memberUID: uid=alice,ou=People,dc=cluster,dc=local + memberUID: uid=bob,ou=People,dc=cluster,dc=local + + dn: cn=blue,ou=Groups,dc=cluster,dc=local + objectclass: top + objectclass: posixGroup + gidNumber: 419 + cn: blue + description: Blue Team + memberUID: uid=bob,ou=People,dc=cluster,dc=local + + dn: cn=red,ou=Groups,dc=cluster,dc=local + objectclass: top + objectclass: posixGroup + gidNumber: 420 + cn: red + description: Red Team + memberUID: uid=alice,ou=People,dc=cluster,dc=local + +secrets: + identity: + admin: admin + ldap: ldap + oci_image_registry: + ldap: ldap-oci-image-registry-key + +openldap: + domain: cluster.local + password: password + +manifests: + configmap_bin: true + configmap_etc: true + job_bootstrap: true + job_image_repo_sync: true + network_policy: false + secret_registry: true + statefulset: true + service: true +... diff --git a/libvirt/.helmignore b/libvirt/.helmignore new file mode 100644 index 0000000000..b54c347b85 --- /dev/null +++ b/libvirt/.helmignore @@ -0,0 +1 @@ +values_overrides diff --git a/libvirt/Chart.yaml b/libvirt/Chart.yaml new file mode 100644 index 0000000000..604072232c --- /dev/null +++ b/libvirt/Chart.yaml @@ -0,0 +1,29 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: v1.0.0 +description: OpenStack-Helm libvirt +name: libvirt +version: 2024.2.0 +home: https://libvirt.org +sources: + - https://libvirt.org/git/?p=libvirt.git;a=summary + - https://opendev.org/openstack/openstack-helm +maintainers: + - name: OpenStack-Helm Authors +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit + version: ">= 0.1.0" +... diff --git a/libvirt/templates/bin/_ceph-admin-keyring.sh.tpl b/libvirt/templates/bin/_ceph-admin-keyring.sh.tpl new file mode 100644 index 0000000000..8c36d4b088 --- /dev/null +++ b/libvirt/templates/bin/_ceph-admin-keyring.sh.tpl @@ -0,0 +1,29 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +export HOME=/tmp + +cat > /etc/ceph/ceph.client.admin.keyring << EOF +[client.admin] +{{- if .Values.conf.ceph.admin_keyring }} + key = {{ .Values.conf.ceph.admin_keyring }} +{{- else }} + key = $(cat /tmp/client-keyring) +{{- end }} +EOF + +exit 0 diff --git a/libvirt/templates/bin/_ceph-keyring.sh.tpl b/libvirt/templates/bin/_ceph-keyring.sh.tpl new file mode 100644 index 0000000000..35f5c111b3 --- /dev/null +++ b/libvirt/templates/bin/_ceph-keyring.sh.tpl @@ -0,0 +1,51 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +export HOME=/tmp + +cp -fv /etc/ceph/ceph.conf.template /etc/ceph/ceph.conf + +KEYRING=/etc/ceph/ceph.client.${CEPH_CINDER_USER}.keyring +{{- if .Values.conf.ceph.cinder.keyring }} +cat > ${KEYRING} </dev/null | grep -w libvirtd)" ]; then + set +x + for proc in $(ls /proc/*/comm 2>/dev/null); do + if [ "x$(cat $proc 2>/dev/null | grep -w libvirtd)" == "xlibvirtd" ]; then + set -x + libvirtpid=$(echo $proc | cut -f 3 -d '/') + echo "WARNING: libvirtd daemon already running on host" 1>&2 + echo "$(cat "/proc/${libvirtpid}/status" 2>/dev/null | grep State)" 1>&2 + kill -9 "$libvirtpid" || true + set +x + fi + done + set -x +fi + +rm -f /var/run/libvirtd.pid + +if [[ -c /dev/kvm ]]; then + chmod 660 /dev/kvm + chown root:kvm /dev/kvm +fi + +#Setup Cgroups to use when breaking out of Kubernetes defined groups +CGROUPS="" +for CGROUP in {{ .Values.conf.kubernetes.cgroup_controllers | include "helm-toolkit.utils.joinListWithSpace" }}; do + if [ -d /sys/fs/cgroup/${CGROUP} ] || grep -w $CGROUP /sys/fs/cgroup/cgroup.controllers; then + CGROUPS+="${CGROUP}," + fi +done +cgcreate -g ${CGROUPS%,}:/osh-libvirt + +# We assume that if hugepage count > 0, then hugepages should be exposed to libvirt/qemu +hp_count="$(cat /proc/meminfo | grep HugePages_Total | tr -cd '[:digit:]')" +if [ 0"$hp_count" -gt 0 ]; then + + echo "INFO: Detected hugepage count of '$hp_count'. Enabling hugepage settings for libvirt/qemu." + + # Enable KVM hugepages for QEMU + if [ -n "$(grep KVM_HUGEPAGES=0 /etc/default/qemu-kvm)" ]; then + sed -i 's/.*KVM_HUGEPAGES=0.*/KVM_HUGEPAGES=1/g' /etc/default/qemu-kvm + else + echo KVM_HUGEPAGES=1 >> /etc/default/qemu-kvm + fi + + # Ensure that the hugepage mount location is available/mapped inside the + # container. This assumes use of the default ubuntu dev-hugepages.mount + # systemd unit which mounts hugepages at this location. + if [ ! -d /dev/hugepages ]; then + echo "ERROR: Hugepages configured in kernel, but libvirtd container cannot access /dev/hugepages" + exit 1 + fi +fi + +if [ -n "${LIBVIRT_CEPH_CINDER_SECRET_UUID}" ] || [ -n "${LIBVIRT_EXTERNAL_CEPH_CINDER_SECRET_UUID}" ] ; then + + cgexec -g ${CGROUPS%,}:/osh-libvirt systemd-run --scope --slice=system libvirtd --listen & + + tmpsecret=$(mktemp --suffix .xml) + if [ -n "${LIBVIRT_EXTERNAL_CEPH_CINDER_SECRET_UUID}" ] ; then + tmpsecret2=$(mktemp --suffix .xml) + fi + function cleanup { + rm -f "${tmpsecret}" + if [ -n "${LIBVIRT_EXTERNAL_CEPH_CINDER_SECRET_UUID}" ] ; then + rm -f "${tmpsecret2}" + fi + } + trap cleanup EXIT + + # Wait for the libvirtd is up + TIMEOUT=60 + while [[ ! -f /var/run/libvirtd.pid ]]; do + if [[ ${TIMEOUT} -gt 0 ]]; then + let TIMEOUT-=1 + sleep 1 + else + echo "ERROR: libvirt did not start in time (pid file missing)" + exit 1 + fi + done + + # Even though we see the pid file the socket immediately (this is + # needed for virsh) + TIMEOUT=10 + while [[ ! -e /var/run/libvirt/libvirt-sock ]]; do + if [[ ${TIMEOUT} -gt 0 ]]; then + let TIMEOUT-=1 + sleep 1 + else + echo "ERROR: libvirt did not start in time (socket missing)" + exit 1 + fi + done + + function create_virsh_libvirt_secret { + sec_user=$1 + sec_uuid=$2 + sec_ceph_keyring=$3 + cat > ${tmpsecret} < + ${sec_uuid} + + client.${sec_user}. secret + + +EOF + virsh secret-define --file ${tmpsecret} + virsh secret-set-value --secret "${sec_uuid}" --base64 "${sec_ceph_keyring}" + } + + if [ -z "${CEPH_CINDER_KEYRING}" ] && [ -n "${CEPH_CINDER_USER}" ] ; then + CEPH_CINDER_KEYRING=$(awk '/key/{print $3}' /etc/ceph/ceph.client.${CEPH_CINDER_USER}.keyring) + fi + if [ -n "${CEPH_CINDER_USER}" ] ; then + create_virsh_libvirt_secret ${CEPH_CINDER_USER} ${LIBVIRT_CEPH_CINDER_SECRET_UUID} ${CEPH_CINDER_KEYRING} + fi + + if [ -n "${LIBVIRT_EXTERNAL_CEPH_CINDER_SECRET_UUID}" ] ; then + EXTERNAL_CEPH_CINDER_KEYRING=$(cat /tmp/external-ceph-client-keyring) + create_virsh_libvirt_secret ${EXTERNAL_CEPH_CINDER_USER} ${LIBVIRT_EXTERNAL_CEPH_CINDER_SECRET_UUID} ${EXTERNAL_CEPH_CINDER_KEYRING} + fi + + cleanup + + # stop libvirtd; we needed it up to create secrets + LIBVIRTD_PID=$(cat /var/run/libvirtd.pid) + kill $LIBVIRTD_PID + tail --pid=$LIBVIRTD_PID -f /dev/null + +fi + +# NOTE(vsaienko): changing CGROUP is required as restart of the pod will cause domains restarts +cgexec -g ${CGROUPS%,}:/osh-libvirt systemd-run --scope --slice=system libvirtd --listen diff --git a/libvirt/templates/configmap-apparmor.yaml b/libvirt/templates/configmap-apparmor.yaml new file mode 100644 index 0000000000..a13e3c48fc --- /dev/null +++ b/libvirt/templates/configmap-apparmor.yaml @@ -0,0 +1,15 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- dict "envAll" . "component" "libvirt" | include "helm-toolkit.snippets.kubernetes_apparmor_configmap" }} diff --git a/libvirt/templates/configmap-bin.yaml b/libvirt/templates/configmap-bin.yaml new file mode 100644 index 0000000000..22e99db50c --- /dev/null +++ b/libvirt/templates/configmap-bin.yaml @@ -0,0 +1,47 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "libvirt.configmap.bin" }} +{{- $configMapName := index . 0 }} +{{- $envAll := index . 1 }} +{{- with $envAll }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $configMapName }} +data: +{{- if .Values.images.local_registry.active }} + image-repo-sync.sh: | +{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} +{{- end }} + libvirt.sh: | +{{ tuple "bin/_libvirt.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- if eq .Values.conf.qemu.vnc_tls "1" }} + cert-init.sh: | +{{ tpl .Values.conf.vencrypt.cert_init_sh . | indent 4 }} +{{- end }} +{{- if .Values.conf.ceph.enabled }} + ceph-keyring.sh: | +{{ tuple "bin/_ceph-keyring.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + ceph-admin-keyring.sh: | +{{ tuple "bin/_ceph-admin-keyring.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} +{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.init_modules.script "key" "libvirt-init-modules.sh") | indent 2 }} +{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.dynamic_options.script "key" "init-dynamic-options.sh") | indent 2 }} +{{- end }} +{{- end }} +{{- if .Values.manifests.configmap_bin }} +{{- list "libvirt-bin" . | include "libvirt.configmap.bin" }} +{{- end }} diff --git a/libvirt/templates/configmap-etc.yaml b/libvirt/templates/configmap-etc.yaml new file mode 100644 index 0000000000..68ce576b31 --- /dev/null +++ b/libvirt/templates/configmap-etc.yaml @@ -0,0 +1,33 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "libvirt.configmap.etc" }} +{{- $configMapName := index . 0 }} +{{- $envAll := index . 1 }} +{{- with $envAll }} + +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $configMapName }} +type: Opaque +data: + qemu.conf: {{ include "libvirt.utils.to_libvirt_conf" .Values.conf.qemu | b64enc }} +{{- end }} +{{- end }} + +{{- if .Values.manifests.configmap_etc }} +{{- list "libvirt-etc" . | include "libvirt.configmap.etc" }} +{{- end }} diff --git a/libvirt/templates/daemonset-libvirt.yaml b/libvirt/templates/daemonset-libvirt.yaml new file mode 100644 index 0000000000..48c16b04c0 --- /dev/null +++ b/libvirt/templates/daemonset-libvirt.yaml @@ -0,0 +1,414 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "libvirtProbeTemplate" }} +exec: + command: + - bash + - -c + - /usr/bin/virsh connect +{{- end }} + +{{- define "libvirt.daemonset" }} +{{- $daemonset := index . 0 }} +{{- $configMapName := index . 1 }} +{{- $serviceAccountName := index . 2 }} +{{- $envAll := index . 3 }} +{{- $ssl_enabled := false }} +{{- if eq $envAll.Values.conf.libvirt.listen_tls "1" }} +{{- $ssl_enabled = true }} +{{- end }} +{{- with $envAll }} + +{{- $mounts_libvirt := .Values.pod.mounts.libvirt.libvirt }} +{{- $mounts_libvirt_init := .Values.pod.mounts.libvirt.init_container }} + +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: libvirt + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll .Chart.Name $daemonset | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + selector: + matchLabels: +{{ tuple $envAll .Chart.Name $daemonset | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} +{{ tuple $envAll $daemonset | include "helm-toolkit.snippets.kubernetes_upgrades_daemonset" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll .Chart.Name $daemonset | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{- dict "envAll" $envAll "podName" "libvirt-libvirt-default" "containerNames" (list "libvirt") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} + spec: +{{ dict "envAll" $envAll "application" "libvirt" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + nodeSelector: + {{ .Values.labels.agent.libvirt.node_selector_key }}: {{ .Values.labels.agent.libvirt.node_selector_value }} +{{ if $envAll.Values.pod.tolerations.libvirt.enabled }} +{{ tuple $envAll "libvirt" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }} +{{ end }} + hostNetwork: true + hostPID: true + hostIPC: true + dnsPolicy: {{ .Values.pod.dns_policy }} + initContainers: +{{ tuple $envAll "pod_dependency" $mounts_libvirt_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} +{{ dict "envAll" $envAll | include "helm-toolkit.snippets.kubernetes_apparmor_loader_init_container" | indent 8 }} +{{- if .Values.conf.init_modules.enabled }} + - name: libvirt-init-modules +{{ tuple $envAll "libvirt" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ dict "envAll" $envAll "application" "libvirt" "container" "libvirt_init_modules" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + terminationMessagePath: /var/log/termination-log + command: + - /tmp/libvirt-init-modules.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: etc-modprobe-d + mountPath: /etc/modprobe.d_host + - name: host-rootfs + mountPath: /mnt/host-rootfs + mountPropagation: HostToContainer + readOnly: true + - name: libvirt-bin + mountPath: /tmp/libvirt-init-modules.sh + subPath: libvirt-init-modules.sh + readOnly: true +{{- end }} + - name: init-dynamic-options +{{ tuple $envAll "libvirt" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ dict "envAll" $envAll "application" "libvirt" "container" "init_dynamic_options" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + terminationMessagePath: /var/log/termination-log + command: + - /tmp/init-dynamic-options.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: pod-shared + mountPath: /tmp/pod-shared + - name: libvirt-bin + mountPath: /tmp/init-dynamic-options.sh + subPath: init-dynamic-options.sh + readOnly: true +{{- if eq .Values.conf.qemu.vnc_tls "1" }} + - name: cert-init-vnc +{{ tuple $envAll "kubectl" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ dict "envAll" $envAll "application" "libvirt" "container" "cert_init" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/cert-init.sh + env: + - name: TYPE + value: vnc + - name: ISSUER_KIND + value: {{ .Values.conf.vencrypt.issuer.kind }} + - name: ISSUER_NAME + value: {{ .Values.conf.vencrypt.issuer.name }} + - name: POD_UID + valueFrom: + fieldRef: + fieldPath: metadata.uid + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: libvirt-bin + mountPath: /tmp/cert-init.sh + subPath: cert-init.sh + readOnly: true +{{- end }} +{{- if .Values.conf.ceph.enabled }} + {{- if empty .Values.conf.ceph.cinder.keyring }} + - name: ceph-admin-keyring-placement +{{ tuple $envAll "ceph_config_helper" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ dict "envAll" $envAll "application" "libvirt" "container" "ceph_admin_keyring_placement" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/ceph-admin-keyring.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: etcceph + mountPath: /etc/ceph + - name: libvirt-bin + mountPath: /tmp/ceph-admin-keyring.sh + subPath: ceph-admin-keyring.sh + readOnly: true + {{- if empty .Values.conf.ceph.admin_keyring }} + - name: ceph-keyring + mountPath: /tmp/client-keyring + subPath: key + readOnly: true + {{ end }} + {{ end }} + - name: ceph-keyring-placement +{{ tuple $envAll "ceph_config_helper" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ dict "envAll" $envAll "application" "libvirt" "container" "ceph_keyring_placement" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + env: + - name: CEPH_CINDER_USER + value: "{{ .Values.conf.ceph.cinder.user }}" + {{- if .Values.conf.ceph.cinder.keyring }} + - name: CEPH_CINDER_KEYRING + value: "{{ .Values.conf.ceph.cinder.keyring }}" + {{ end }} + - name: LIBVIRT_CEPH_CINDER_SECRET_UUID + value: "{{ .Values.conf.ceph.cinder.secret_uuid }}" + command: + - /tmp/ceph-keyring.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: etcceph + mountPath: /etc/ceph + - name: libvirt-bin + mountPath: /tmp/ceph-keyring.sh + subPath: ceph-keyring.sh + readOnly: true + - name: ceph-etc + mountPath: /etc/ceph/ceph.conf.template + subPath: ceph.conf + readOnly: true +{{- end }} + containers: + - name: libvirt +{{ tuple $envAll "libvirt" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.libvirt | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "libvirt" "container" "libvirt" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + env: + {{- if .Values.conf.ceph.enabled }} + - name: CEPH_CINDER_USER + value: "{{ .Values.conf.ceph.cinder.user }}" + {{- if .Values.conf.ceph.cinder.keyring }} + - name: CEPH_CINDER_KEYRING + value: "{{ .Values.conf.ceph.cinder.keyring }}" + {{ end }} + - name: LIBVIRT_CEPH_CINDER_SECRET_UUID + value: "{{ .Values.conf.ceph.cinder.secret_uuid }}" + {{ end }} + {{- if .Values.conf.ceph.cinder.external_ceph.enabled }} + - name: EXTERNAL_CEPH_CINDER_USER + value: "{{ .Values.conf.ceph.cinder.external_ceph.user }}" + - name: LIBVIRT_EXTERNAL_CEPH_CINDER_SECRET_UUID + value: "{{ .Values.conf.ceph.cinder.external_ceph.secret_uuid }}" + {{ end }} +{{ dict "envAll" . "component" "libvirt" "container" "libvirt" "type" "readiness" "probeTemplate" (include "libvirtProbeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }} +{{ dict "envAll" . "component" "libvirt" "container" "libvirt" "type" "liveness" "probeTemplate" (include "libvirtProbeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }} + command: + - /tmp/libvirt.sh + lifecycle: + preStop: + exec: + command: + - bash + - -c + - |- + kill $(cat /var/run/libvirtd.pid) + volumeMounts: + {{ dict "enabled" $ssl_enabled "name" "ssl-client" "path" "/etc/pki/libvirt" "certs" (tuple "clientcert.pem" "clientkey.pem" ) | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }} + {{ dict "enabled" $ssl_enabled "name" "ssl-server-cert" "path" "/etc/pki/libvirt" "certs" (tuple "servercert.pem" ) | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }} + {{ dict "enabled" $ssl_enabled "name" "ssl-server-key" "path" "/etc/pki/libvirt/private" "certs" (tuple "serverkey.pem" ) | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }} + {{ dict "enabled" $ssl_enabled "name" "ssl-ca-cert" "path" "/etc/pki/CA" "certs" (tuple "cacert.pem" ) | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }} + - name: pod-tmp + mountPath: /tmp + - name: libvirt-bin + mountPath: /tmp/libvirt.sh + subPath: libvirt.sh + readOnly: true + - name: pod-shared + mountPath: /etc/libvirt/libvirtd.conf + subPath: libvirtd.conf + readOnly: true + - name: libvirt-etc + mountPath: /etc/libvirt/qemu.conf + subPath: qemu.conf + readOnly: true + - name: etc-libvirt-qemu + mountPath: /etc/libvirt/qemu + - mountPath: /lib/modules + name: libmodules + readOnly: true + - name: var-lib-libvirt + mountPath: /var/lib/libvirt + {{- if or ( gt .Capabilities.KubeVersion.Major "1" ) ( ge .Capabilities.KubeVersion.Minor "10" ) }} + mountPropagation: Bidirectional + {{- end }} + - name: var-lib-nova + mountPath: /var/lib/nova + {{- if or ( gt .Capabilities.KubeVersion.Major "1" ) ( ge .Capabilities.KubeVersion.Minor "10" ) }} + mountPropagation: Bidirectional + {{- end }} + - name: run + mountPath: /run + - name: dev + mountPath: /dev + - name: cgroup + mountPath: /sys/fs/cgroup + - name: logs + mountPath: /var/log/libvirt + - name: machine-id + mountPath: /etc/machine-id + readOnly: true + {{- if .Values.conf.ceph.enabled }} + - name: etcceph + mountPath: /etc/ceph + {{- if or ( gt .Capabilities.KubeVersion.Major "1" ) ( ge .Capabilities.KubeVersion.Minor "10" ) }} + mountPropagation: Bidirectional + {{- end }} + {{- if empty .Values.conf.ceph.cinder.keyring }} + - name: ceph-keyring + mountPath: /tmp/client-keyring + subPath: key + readOnly: true + {{- end }} + {{- end }} + {{- if .Values.conf.ceph.cinder.external_ceph.enabled }} + - name: external-ceph-keyring + mountPath: /tmp/external-ceph-client-keyring + subPath: key + readOnly: true + {{- end }} +{{ if $mounts_libvirt.volumeMounts }}{{ toYaml $mounts_libvirt.volumeMounts | indent 12 }}{{ end }} + {{- if .Values.pod.sidecars.libvirt_exporter }} + - name: libvirt-exporter +{{ tuple $envAll "libvirt_exporter" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.libvirt_exporter | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "libvirt" "container" "libvirt_exporter" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + args: + - "--libvirt.nova" + ports: + - name: metrics + protocol: TCP + containerPort: {{ tuple "libvirt_exporter" "direct" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + livenessProbe: + httpGet: + path: / + port: metrics + readinessProbe: + httpGet: + path: / + port: metrics + volumeMounts: + - name: run + mountPath: /run + {{- if or ( gt .Capabilities.KubeVersion.Major "1" ) ( ge .Capabilities.KubeVersion.Minor "10" ) }} + mountPropagation: Bidirectional + {{- end }} + {{- end }} + volumes: + {{ dict "enabled" $ssl_enabled "secretName" $envAll.Values.secrets.tls.client "name" "ssl-client" "path" "/etc/pki/libvirt" "certs" (tuple "clientcert.pem" "clientkey.pem" ) | include "helm-toolkit.snippets.tls_volume" | indent 8 }} + {{ dict "enabled" $ssl_enabled "secretName" $envAll.Values.secrets.tls.server "name" "ssl-server-cert" "path" "/etc/pki/libvirt" "certs" (tuple "servercert.pem" ) | include "helm-toolkit.snippets.tls_volume" | indent 8 }} + {{ dict "enabled" $ssl_enabled "secretName" $envAll.Values.secrets.tls.server "name" "ssl-server-key" "path" "/etc/pki/libvirt/private" "certs" (tuple "serverkey.pem" ) | include "helm-toolkit.snippets.tls_volume" | indent 8 }} + {{ dict "enabled" $ssl_enabled "secretName" $envAll.Values.secrets.tls.server "name" "ssl-ca-cert" "path" "/etc/pki/CA" "certs" (tuple "cacert.pem" ) | include "helm-toolkit.snippets.tls_volume" | indent 8 }} + - name: pod-tmp + emptyDir: {} + - name: libvirt-bin + configMap: + name: libvirt-bin + defaultMode: 0555 + - name: libvirt-etc + secret: + secretName: {{ $configMapName }} + defaultMode: 0444 + {{- if .Values.conf.ceph.enabled }} + - name: etcceph + hostPath: + path: /var/lib/openstack-helm/compute/libvirt + - name: ceph-etc + configMap: + name: {{ .Values.ceph_client.configmap }} + defaultMode: 0444 + {{- if empty .Values.conf.ceph.cinder.keyring }} + - name: ceph-keyring + secret: + secretName: {{ .Values.ceph_client.user_secret_name }} + {{ end }} + {{ end }} + {{- if .Values.conf.ceph.cinder.external_ceph.enabled }} + - name: external-ceph-keyring + secret: + secretName: {{ .Values.conf.ceph.cinder.external_ceph.user_secret_name }} + {{ end }} + - name: libmodules + hostPath: + path: /lib/modules + - name: var-lib-libvirt + hostPath: + path: /var/lib/libvirt + - name: var-lib-nova + hostPath: + path: /var/lib/nova + - name: run + hostPath: + path: /run + - name: dev + hostPath: + path: /dev + - name: logs + hostPath: + path: /var/log/libvirt + - name: cgroup + hostPath: + path: /sys/fs/cgroup + - name: machine-id + hostPath: + path: /etc/machine-id + - name: etc-libvirt-qemu + hostPath: + path: /etc/libvirt/qemu + - name: etc-modprobe-d + hostPath: + path: /etc/modprobe.d + - name: host-rootfs + hostPath: + path: / + type: Directory + - name: pod-shared + emptyDir: {} +{{ dict "envAll" $envAll "component" "libvirt" "requireSys" true | include "helm-toolkit.snippets.kubernetes_apparmor_volumes" | indent 8 }} +{{ if $mounts_libvirt.volumes }}{{ toYaml $mounts_libvirt.volumes | indent 8 }}{{ end }} +{{- end }} +{{- end }} + +{{- if .Values.manifests.daemonset_libvirt }} + +{{- $envAll := . }} +{{- $daemonset := "libvirt" }} +{{- $configMapName := "libvirt-etc" }} +{{- $serviceAccountName := "libvirt" }} + +{{- $dependencyOpts := dict "envAll" $envAll "dependencyMixinParam" $envAll.Values.network.backend "dependencyKey" "libvirt" -}} +{{- $_ := include "helm-toolkit.utils.dependency_resolver" $dependencyOpts | toString | fromYaml }} + +{{ tuple $envAll "pod_dependency" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +{{- $configmap_yaml := "libvirt.configmap.etc" }} + +{{/* Preffer using .Values.overrides rather than .Values.conf.overrides */}} +{{- list $daemonset "libvirt.daemonset" $serviceAccountName $configmap_yaml $configMapName "libvirt.configmap.bin" "libvirt-bin" . | include "helm-toolkit.utils.daemonset_overrides_root" }} +{{- end }} diff --git a/libvirt/templates/job-image-repo-sync.yaml b/libvirt/templates/job-image-repo-sync.yaml new file mode 100644 index 0000000000..91d52820c9 --- /dev/null +++ b/libvirt/templates/job-image-repo-sync.yaml @@ -0,0 +1,21 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "libvirt" -}} +{{- if .Values.pod.tolerations.libvirt.enabled -}} +{{- $_ := set $imageRepoSyncJob "tolerationsEnabled" true -}} +{{- end -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/libvirt/templates/network-policy.yaml b/libvirt/templates/network-policy.yaml new file mode 100644 index 0000000000..6ed51aaafc --- /dev/null +++ b/libvirt/templates/network-policy.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.network_policy -}} +{{- $netpol_opts := dict "envAll" . "name" "application" "label" "libvirt" -}} +{{ $netpol_opts | include "helm-toolkit.manifests.kubernetes_network_policy" }} +{{- end -}} diff --git a/libvirt/templates/role-cert-manager.yaml b/libvirt/templates/role-cert-manager.yaml new file mode 100755 index 0000000000..b830690c19 --- /dev/null +++ b/libvirt/templates/role-cert-manager.yaml @@ -0,0 +1,54 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.role_cert_manager }} +{{- $serviceAccountName := "libvirt" }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ .Release.Name }}-cert-manager + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ .Release.Name }}-cert-manager +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ .Release.Name }}-cert-manager + namespace: {{ .Release.Namespace }} +rules: + - apiGroups: + - cert-manager.io + verbs: + - get + - list + - create + - watch + resources: + - certificates + - apiGroups: + - "" + verbs: + - get + - patch + resources: + - secrets +{{- end -}} \ No newline at end of file diff --git a/libvirt/templates/secret-registry.yaml b/libvirt/templates/secret-registry.yaml new file mode 100644 index 0000000000..da979b3223 --- /dev/null +++ b/libvirt/templates/secret-registry.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_registry .Values.endpoints.oci_image_registry.auth.enabled }} +{{ include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) }} +{{- end }} diff --git a/libvirt/templates/utils/_to_libvirt_conf.tpl b/libvirt/templates/utils/_to_libvirt_conf.tpl new file mode 100644 index 0000000000..31e097817b --- /dev/null +++ b/libvirt/templates/utils/_to_libvirt_conf.tpl @@ -0,0 +1,51 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{/* +abstract: | + Builds a libvirt compatible config file. +values: | + conf: + libvirt: + log_level: 3 + cgroup_controllers: + - cpu + - cpuacct +usage: | + {{ include "libvirt.utils.to_libvirt_conf" .Values.conf.libvirt }} +return: | + cgroup_controllers = [ "cpu", "cpuacct" ] + log_level = 3 +*/}} + +{{- define "libvirt.utils._to_libvirt_conf.list_to_string" -}} +{{- $local := dict "first" true -}} +{{- range $k, $v := . -}}{{- if not $local.first -}}, {{ end -}}{{- $v | quote -}}{{- $_ := set $local "first" false -}}{{- end -}} +{{- end -}} + +{{- define "libvirt.utils.to_libvirt_conf" -}} +{{- range $key, $value := . -}} +{{- if kindIs "slice" $value }} +{{ $key }} = [ {{ include "libvirt.utils._to_libvirt_conf.list_to_string" $value }} ] +{{- else if kindIs "string" $value }} +{{- if regexMatch "^[0-9]+$" $value }} +{{ $key }} = {{ $value }} +{{- else }} +{{ $key }} = {{ $value | quote }} +{{- end }} +{{- else }} +{{ $key }} = {{ $value }} +{{- end }} +{{- end -}} +{{- end -}} diff --git a/libvirt/values.yaml b/libvirt/values.yaml new file mode 100644 index 0000000000..69efc4a198 --- /dev/null +++ b/libvirt/values.yaml @@ -0,0 +1,413 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for libvirt. +# This is a YAML-formatted file. +# Declare name/value pairs to be passed into your templates. +# name: value + +--- +release_group: null + +labels: + agent: + libvirt: + node_selector_key: openstack-compute-node + node_selector_value: enabled + +images: + tags: + libvirt: docker.io/openstackhelm/libvirt:latest-ubuntu_focal + libvirt_exporter: vexxhost/libvirtd-exporter:latest + ceph_config_helper: 'docker.io/openstackhelm/ceph-config-helper:ubuntu_jammy_19.2.1-1-20250207' + dep_check: quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal + image_repo_sync: docker.io/library/docker:17.07.0 + kubectl: docker.io/bitnami/kubectl:latest + pull_policy: "IfNotPresent" + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +network: + # provide what type of network wiring will be used + # possible options: openvswitch, linuxbridge, sriov + backend: + - openvswitch + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + libvirt: + username: libvirt + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + libvirt_exporter: + port: + metrics: + default: 9474 + +network_policy: + libvirt: + ingress: + - {} + egress: + - {} + +ceph_client: + configmap: ceph-etc + user_secret_name: pvc-ceph-client-key + +conf: + ceph: + enabled: true + admin_keyring: null + cinder: + user: "cinder" + keyring: null + secret_uuid: 457eb676-33da-42ec-9a8c-9293d545c337 + # Cinder Ceph backend that is not configured by the k8s cluter + external_ceph: + enabled: false + user: null + secret_uuid: null + user_secret_name: null + libvirt: + listen_tcp: "1" + listen_tls: "0" + auth_tcp: "none" + ca_file: "/etc/pki/CA/cacert.pem" + cert_file: "/etc/pki/libvirt/servercert.pem" + key_file: "/etc/pki/libvirt/private/serverkey.pem" + auth_unix_rw: "none" + listen_addr: "${LISTEN_IP_ADDRESS}" + log_level: "3" + log_outputs: "1:file:/var/log/libvirt/libvirtd.log" + # Modifies the config in which value is specified as the name of a variable + # that is computed in the script. + dynamic_options: + libvirt: + listen_interface: null + listen_address: 127.0.0.1 + script: | + #!/bin/bash + set -ex + + LIBVIRT_CONF_PATH=/tmp/pod-shared/libvirtd.conf + + {{- if .Values.conf.dynamic_options.libvirt.listen_interface }} + + LISTEN_INTERFACE="{{ .Values.conf.dynamic_options.libvirt.listen_interface }}" + LISTEN_IP_ADDRESS=$(ip address show $LISTEN_INTERFACE | grep 'inet ' | awk '{print $2}' | awk -F "/" '{print $1}') + {{- else if .Values.conf.dynamic_options.libvirt.listen_address }} + LISTEN_IP_ADDRESS={{ .Values.conf.dynamic_options.libvirt.listen_address }} + {{- end }} + + if [[ -z $LISTEN_IP_ADDRESS ]]; then + echo "LISTEN_IP_ADDRESS is not set." + exit 1 + fi + + tee > ${LIBVIRT_CONF_PATH} << EOF + {{ include "libvirt.utils.to_libvirt_conf" .Values.conf.libvirt }} + EOF + qemu: + vnc_tls: "0" + vnc_tls_x509_verify: "0" + stdio_handler: "file" + user: "nova" + group: "kvm" + kubernetes: + cgroup: "kubepods.slice" + # List of cgroup controller we want to use when breaking out of + # Kubernetes defined groups + cgroup_controllers: + - blkio + - cpu + - devices + - freezer + - hugetlb + - memory + - net_cls + - perf_event + - rdma + - misc + - pids + init_modules: + enabled: false + script: | + #!/bin/bash + + set -ex + export HOME=/tmp + KVM_QEMU_CONF_HOST="/etc/modprobe.d_host/qemu-system-x86.conf" + + if [[ ! -f "${KVM_QEMU_CONF_HOST}" ]]; then + if grep vmx /proc/cpuinfo; then + cat << EOF > ${KVM_QEMU_CONF_HOST} + options kvm_intel nested=1 + options kvm_intel enable_apicv=1 + options kvm_intel ept=1 + EOF + modprobe -r kvm_intel || true + modprobe kvm_intel nested=1 + elif grep svm /proc/cpuinfo; then + cat << EOF > ${KVM_QEMU_CONF_HOST} + options kvm_amd nested=1 + EOF + modprobe -r kvm_amd || true + modprobe kvm_amd nested=1 + else + echo "Nested virtualization is not supported" + fi + fi + vencrypt: + # Issuer to use for the vencrypt certs. + issuer: + kind: ClusterIssuer + name: ca-clusterissuer + # Script is included here (vs in bin/) to allow overriding, in the case that + # communication happens over an IP other than the pod IP for some reason. + cert_init_sh: | + #!/bin/bash + set -x + + HOSTNAME_FQDN=$(hostname --fqdn) + + # Script to create certs for each libvirt pod based on pod IP (by default). + cat < /tmp/${TYPE}.crt + kubectl -n ${POD_NAMESPACE} get secret ${POD_NAME}-${TYPE} -o jsonpath='{.data.tls\.key}' | base64 -d > /tmp/${TYPE}.key + kubectl -n ${POD_NAMESPACE} get secret ${POD_NAME}-${TYPE} -o jsonpath='{.data.ca\.crt}' | base64 -d > /tmp/${TYPE}-ca.crt +pod: + probes: + libvirt: + libvirt: + liveness: + enabled: true + params: + initialDelaySeconds: 30 + periodSeconds: 60 + timeoutSeconds: 5 + readiness: + enabled: true + params: + initialDelaySeconds: 15 + periodSeconds: 60 + timeoutSeconds: 5 + security_context: + libvirt: + pod: + runAsUser: 0 + container: + ceph_admin_keyring_placement: + readOnlyRootFilesystem: false + ceph_keyring_placement: + readOnlyRootFilesystem: false + libvirt: + privileged: true + readOnlyRootFilesystem: false + libvirt_exporter: + privileged: true + libvirt_init_modules: + readOnlyRootFilesystem: true + privileged: true + capabilities: + drop: + - ALL + init_dynamic_options: + runAsUser: 65534 + runAsNonRoot: true + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + sidecars: + libvirt_exporter: false + + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + weight: + default: 10 + tolerations: + libvirt: + enabled: false + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + dns_policy: "ClusterFirstWithHostNet" + mounts: + libvirt: + init_container: null + libvirt: + lifecycle: + upgrades: + daemonsets: + pod_replacement_strategy: RollingUpdate + libvirt: + enabled: true + min_ready_seconds: 0 + max_unavailable: 1 + resources: + enabled: false + libvirt: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + jobs: + image_repo_sync: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + libvirt_exporter: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "256Mi" + cpu: "500m" + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - libvirt-image-repo-sync + services: + - endpoint: node + service: local_image_registry + targeted: + ovn: + libvirt: + pod: + - requireSameNode: true + labels: + application: ovn + component: ovn-controller + openvswitch: + libvirt: + pod: + - requireSameNode: true + labels: + application: neutron + component: neutron-ovs-agent + linuxbridge: + libvirt: + pod: + - requireSameNode: true + labels: + application: neutron + component: neutron-lb-agent + sriov: + libvirt: + pod: + - requireSameNode: true + labels: + application: neutron + component: neutron-sriov-agent + static: + libvirt: + services: null + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + +manifests: + configmap_bin: true + configmap_etc: true + daemonset_libvirt: true + job_image_repo_sync: true + network_policy: false + role_cert_manager: false + secret_registry: true + +secrets: + oci_image_registry: + libvirt: libvirt-oci-image-registry-key + tls: + server: libvirt-tls-server + client: libvirt-tls-client +... diff --git a/local-storage/Chart.yaml b/local-storage/Chart.yaml new file mode 100644 index 0000000000..feb3927b46 --- /dev/null +++ b/local-storage/Chart.yaml @@ -0,0 +1,26 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: v1.0.0 +description: OpenStack-Helm Local Storage +name: local-storage +version: 2024.2.0 +home: https://kubernetes.io/docs/concepts/storage/volumes/#local +maintainers: + - name: OpenStack-Helm Authors +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit + version: ">= 0.1.0" +... diff --git a/local-storage/templates/persistent-volumes.yaml b/local-storage/templates/persistent-volumes.yaml new file mode 100644 index 0000000000..3f283b54ff --- /dev/null +++ b/local-storage/templates/persistent-volumes.yaml @@ -0,0 +1,42 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.persistent_volumes }} +{{- $envAll := . }} +{{- range .Values.conf.persistent_volumes }} +--- +apiVersion: v1 +kind: PersistentVolume +metadata: + name: {{ .name }} + labels: +{{ tuple $envAll "local-storage" $envAll.Release.Name | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + capacity: + storage: {{ .storage_capacity }} + accessModes: {{ .access_modes }} + persistentVolumeReclaimPolicy: {{ .reclaim_policy }} + storageClassName: {{ $envAll.Release.Name }} + local: + path: {{ .local_path }} + nodeAffinity: + required: + nodeSelectorTerms: + - matchExpressions: + - key: {{ $envAll.Values.labels.node_affinity.node_selector_key }} + operator: In + values: + - {{ $envAll.Values.labels.node_affinity.node_selector_value }} +{{- end }} +{{- end }} diff --git a/local-storage/templates/storage-class.yaml b/local-storage/templates/storage-class.yaml new file mode 100644 index 0000000000..3adf858914 --- /dev/null +++ b/local-storage/templates/storage-class.yaml @@ -0,0 +1,26 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.storage_class }} +{{- $envAll := . }} +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: {{ .Release.Name }} + labels: +{{ tuple $envAll "local-storage" $envAll.Release.Name | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +provisioner: kubernetes.io/no-provisioner +volumeBindingMode: WaitForFirstConsumer +{{- end }} diff --git a/local-storage/values.yaml b/local-storage/values.yaml new file mode 100644 index 0000000000..32a41a7882 --- /dev/null +++ b/local-storage/values.yaml @@ -0,0 +1,41 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +labels: + node_affinity: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +conf: + persistent_volumes: + # For each mount path, one PV should be created. + # If there are two mount paths for local storage are available on two nodes, + # then two PVs details should be defined. Example: + # - name: local-pv-1 (name of the Persistent Volume 1) + # reclaimpolicy: Retain (Reclaim Policy for the PV local-pv-1) + # storage_capacity: "100Gi" (Storage capacity of the PV local-pv-1) + # access_modes: [ "ReadWriteOnce" ] (Access mode for the PV local-pv-1) + # local_path: /mnt/disk/vol1 (Mount path of the local disk, local-pv-1 will be created on) + # - name: local-pv-2 (name of the Persistent Volume 2) + # reclaimpolicy: Retain (Reclaim Policy for the PV local-pv-2) + # storage_capacity: "100Gi" (Storage capacity of the PV local-pv-2) + # access_modes: [ "ReadWriteOnce" ] (Access mode for the PV local-pv-2) + # local_path: /mnt/disk/vol2 (Mount path of the local disk, local-pv-2 will be created on) + # Similarly if three nodes each have disk mount path /var/lib/kubernetes + # which will be acting as local storage for each node, then Persistentvolumes + # should be updated with three entries. + +manifests: + storage_class: true + persistent_volumes: true +... diff --git a/local-volume-provisioner/Chart.yaml b/local-volume-provisioner/Chart.yaml new file mode 100644 index 0000000000..fc763b6cbd --- /dev/null +++ b/local-volume-provisioner/Chart.yaml @@ -0,0 +1,28 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: v1.0.0 +description: OpenStack-Helm local-volume-provisioner +name: local-volume-provisioner +version: 2024.2.0 +home: https://github.com/kubernetes-sigs/sig-storage-local-static-provisioner +sources: + - https://opendev.org/openstack/openstack-helm +maintainers: + - name: OpenStack-Helm Authors +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit + version: ">= 0.1.0" +... diff --git a/local-volume-provisioner/templates/bin/_fakemount.py.tpl b/local-volume-provisioner/templates/bin/_fakemount.py.tpl new file mode 100644 index 0000000000..e9a937f4e2 --- /dev/null +++ b/local-volume-provisioner/templates/bin/_fakemount.py.tpl @@ -0,0 +1,377 @@ +#!/usr/bin/env python3 +# +# Copyright 2019 Mirantis, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +"""Fakemount python module +The module is aimed to crate fake mountpoints (--bind). +Example: + python3 fakemount --config-file '/root/mymount.yml' +Attributes: + config-file - file path to config file that contains fake mounts. +""" +__version__ = "1.0" +import argparse +import logging +import os +import re +import subprocess +import sys +from collections import defaultdict +import yaml +logging.basicConfig(stream=sys.stdout, level=logging.INFO) +LOG = logging.getLogger(__name__) +MOUNT_BIN = "/bin/mount" +###Fork https://github.com/b10011/pyfstab/ ##################################### +# Latest commit 828540d +class InvalidEntry(Exception): + """ + Raised when a string cannot be generated because of the Entry is invalid. + """ +class InvalidFstabLine(Exception): + """ + Raised when a line is invalid in fstab. This doesn't just mean that the + Entry will be invalid but also that the system can not process the fstab + file fully either. + """ +class Entry: + """ + Handles parsing and formatting fstab line entries. + :var device: + (str or None) - + Fstab device (1st parameter in the fstab entry) + :var dir: + (str or None) - + Fstab device (2nd parameter in the fstab entry) + :var type: + (str or None) - + Fstab device (3rd parameter in the fstab entry) + :var options: + (str or None) - + Fstab device (4th parameter in the fstab entry) + :var dump: + (int or None) - + Fstab device (5th parameter in the fstab entry) + :var fsck: + (int or None) - + Fstab device (6th parameter in the fstab entry) + :var valid: + (bool) - + Whether the Entry is valid or not. Can be checked with "if entry:". + """ + def __init__( + self, + _device=None, + _dir=None, + _type=None, + _options=None, + _dump=None, + _fsck=None, + ): + """ + :param _device: Fstab device (1st parameter in the fstab entry) + :type _device: str + :param _dir: Fstab device (2nd parameter in the fstab entry) + :type _dir: str + :param _type: Fstab device (3rd parameter in the fstab entry) + :type _type: str + :param _options: Fstab device (4th parameter in the fstab entry) + :type _options: str + :param _dump: Fstab device (5th parameter in the fstab entry) + :type _dump: int + :param _fsck: Fstab device (6th parameter in the fstab entry) + :type _fsck: int + """ + self.device = _device + self.dir = _dir + self.type = _type + self.options = _options + self.dump = _dump + self.fsck = _fsck + self.valid = True + self.valid &= self.device is not None + self.valid &= self.dir is not None + self.valid &= self.type is not None + self.valid &= self.options is not None + self.valid &= self.dump is not None + self.valid &= self.fsck is not None + def read_string(self, line): + """ + Parses an entry from a string + :param line: Fstab entry line. + :type line: str + :return: self + :rtype: Entry + :raises InvalidEntry: If the data in the string cannot be parsed. + """ + line = line.strip() + if line and not line[0] == "#": + parts = re.split(r"\s+", line) + if len(parts) == 6: + [_device, _dir, _type, _options, _dump, _fsck] = parts + _dump = int(_dump) + _fsck = int(_fsck) + self.device = _device + self.dir = _dir + self.type = _type + self.options = _options + self.dump = _dump + self.fsck = _fsck + self.valid = True + return self + else: + raise InvalidFstabLine() + self.device = None + self.dir = None + self.type = None + self.options = None + self.dump = None + self.fsck = None + self.valid = False + raise InvalidEntry("Entry cannot be parsed") + def write_string(self): + """ + Formats the Entry into fstab entry line. + :return: Fstab entry line. + :rtype: str + :raises InvalidEntry: + A string cannot be generated because the entry is invalid. + """ + if self: + return "{} {} {} {} {} {}".format( + self.device, + self.dir, + self.type, + self.options, + self.dump, + self.fsck, + ) + else: + raise InvalidEntry("Entry cannot be formatted") + def __bool__(self): + return self.valid + def __str__(self): + return self.write_string() + def __repr__(self): + try: + return "".format(str(self)) + except InvalidEntry: + return "" +class Fstab: + """ + Handles reading, parsing, formatting and writing of fstab files. + :var entries: + (list[Entry]) - + List of entries. + When writing to a file, entries are listed from this list. + :var entries_by_device: + (dict[str, list[Entry]]) - + Fstab entries by device. + :var entry_by_dir: + (dict[str, Entry]) - + Fstab entry by directory. + :var entries_by_type: + (dict[str, list[Entry]]) - + Fstab entries by type. + """ + def __init__(self): + self.entries = [] + # A single device can have multiple mountpoints + self.entries_by_device = defaultdict(list) + # If multiple devices have same mountpoint, only the last entry in the + # fstab file is taken into consideration + self.entry_by_dir = dict() + # And the most obvious one, many entries can have mountpoints of same + # type + self.entries_by_type = defaultdict(list) + def read_string(self, data, only_valid=False): + """ + Parses entries from a data string + :param data: Contents of the fstab file + :type data: str + :param only_valid: + Skip the entries that do not actually mount. For example, if device + A is mounted to directory X and later device B is mounted to + directory X, the A mount to X is undone by the system. + :type only_valid: bool + :return: self + :rtype: Fstab + """ + for line in reversed(data.splitlines()): + try: + entry = Entry().read_string(line) + if entry and ( + not only_valid or entry.dir not in self.entry_by_dir + ): + self.entries.insert(0, entry) + self.entries_by_device[entry.device].insert(0, entry) + self.entry_by_dir[entry.dir] = entry + self.entries_by_type[entry.type].insert(0, entry) + except InvalidEntry: + pass + return self + def write_string(self): + """ + Formats entries into a string. + :return: Formatted fstab file. + :rtype: str + :raises InvalidEntry: + A string cannot be generated because one of the entries is invalid. + """ + return "\n".join(str(entry) for entry in self.entries) + def read_file(self, handle, only_valid=False): + """ + Parses entries from a file + :param handle: File handle + :type handle: file + :param only_valid: + Skip the entries that do not actually mount. For example, if device + A is mounted to directory X and later device B is mounted to + directory X, the A mount to X is undone by the system. + :type only_valid: bool + :return: self + :rtype: Fstab + """ + self.read_string(handle.read(), only_valid) + return self + def write_file(self, handle): + """ + Parses entries in data string + :param path: File handle + :type path: file + :return: self + :rtype: Fstab + """ + handle.write(str(self)) + return self + def __bool__(self): + return len(self.entries) > 0 + def __str__(self): + return self.write_string() + def __repr__(self): + res = "= 0.1.0" +... diff --git a/mariadb-backup/README.rst b/mariadb-backup/README.rst new file mode 100644 index 0000000000..4a5c7340b3 --- /dev/null +++ b/mariadb-backup/README.rst @@ -0,0 +1,19 @@ +openstack-helm/mariadb-backup +====================== + +By default, this chart creates a mariadb-backup cronjob that runs in a schedule +in order to create mysql backups. + +This chart depends on mariadb-cluster chart. + +The backups are stored in a PVC and also are possible to upload then to a remote +RGW container. + +You must ensure that your control nodes that should receive mariadb +instances are labeled with ``openstack-control-plane=enabled``, or +whatever you have configured in values.yaml for the label +configuration: + +:: + + kubectl label nodes openstack-control-plane=enabled --all diff --git a/mariadb-backup/templates/bin/_backup_mariadb.sh.tpl b/mariadb-backup/templates/bin/_backup_mariadb.sh.tpl new file mode 100644 index 0000000000..44db641420 --- /dev/null +++ b/mariadb-backup/templates/bin/_backup_mariadb.sh.tpl @@ -0,0 +1,584 @@ +#!/bin/bash + +SCOPE=${1:-"all"} + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +source /tmp/backup_main.sh + +# Export the variables required by the framework +# Note: REMOTE_BACKUP_ENABLED, STORAGE_POLICY and CONTAINER_NAME are already +# exported. +export DB_NAMESPACE=${MARIADB_POD_NAMESPACE} +export DB_NAME="mariadb" +export LOCAL_DAYS_TO_KEEP=${MARIADB_LOCAL_BACKUP_DAYS_TO_KEEP} +export REMOTE_DAYS_TO_KEEP=${MARIADB_REMOTE_BACKUP_DAYS_TO_KEEP} +export REMOTE_BACKUP_RETRIES=${NUMBER_OF_RETRIES_SEND_BACKUP_TO_REMOTE} +export MIN_DELAY_SEND_REMOTE=${MIN_DELAY_SEND_BACKUP_TO_REMOTE} +export MAX_DELAY_SEND_REMOTE=${MAX_DELAY_SEND_BACKUP_TO_REMOTE} +export ARCHIVE_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${DB_NAMESPACE}/${DB_NAME}/archive + +# Dump all the database files to existing $TMP_DIR and save logs to $LOG_FILE +dump_databases_to_directory() { + TMP_DIR=$1 + LOG_FILE=$2 + SCOPE=${3:-"all"} + + + MYSQL="mysql \ + --defaults-file=/etc/mysql/admin_user.cnf \ + --connect-timeout 10" + + MYSQLDUMP="mysqldump \ + --defaults-file=/etc/mysql/admin_user.cnf" + + if [[ "${SCOPE}" == "all" ]]; then + MYSQL_DBNAMES=( $($MYSQL --silent --skip-column-names -e \ + "show databases;" | \ + grep -ivE 'information_schema|performance_schema|mysql|sys') ) + else + if [[ "${SCOPE}" != "information_schema" && "${SCOPE}" != "performance_schema" && "${SCOPE}" != "mysql" && "${SCOPE}" != "sys" ]]; then + MYSQL_DBNAMES=( ${SCOPE} ) + else + log ERROR "It is not allowed to backup database ${SCOPE}." + return 1 + fi + fi + + #check if there is a database to backup, otherwise exit + if [[ -z "${MYSQL_DBNAMES// }" ]] + then + log INFO "There is no database to backup" + return 0 + fi + + #Create a list of Databases + printf "%s\n" "${MYSQL_DBNAMES[@]}" > $TMP_DIR/db.list + + if [[ "${SCOPE}" == "all" ]]; then + #Retrieve and create the GRANT file for all the users +{{- if .Values.manifests.certificates }} + SSL_DSN=";mysql_ssl=1" + SSL_DSN="$SSL_DSN;mysql_ssl_client_key=/etc/mysql/certs/tls.key" + SSL_DSN="$SSL_DSN;mysql_ssl_client_cert=/etc/mysql/certs/tls.crt" + SSL_DSN="$SSL_DSN;mysql_ssl_ca_file=/etc/mysql/certs/ca.crt" + if ! pt-show-grants --defaults-file=/etc/mysql/admin_user.cnf $SSL_DSN \ +{{- else }} + if ! pt-show-grants --defaults-file=/etc/mysql/admin_user.cnf \ +{{- end }} + 2>>"$LOG_FILE" > "$TMP_DIR"/grants.sql; then + log ERROR "Failed to create GRANT for all the users" + return 1 + fi + fi + + #Retrieve and create the GRANT files per DB + for db in "${MYSQL_DBNAMES[@]}" + do + echo $($MYSQL --skip-column-names -e "select concat('show grants for ',user,';') \ + from mysql.db where ucase(db)=ucase('$db');") | \ + sed -r "s/show grants for ([a-zA-Z0-9_-]*)/show grants for '\1'/g" | \ + $MYSQL --silent --skip-column-names 2>>$LOG_FILE > $TMP_DIR/${db}_grant.sql + if [ "$?" -eq 0 ] + then + sed -i 's/$/;/' $TMP_DIR/${db}_grant.sql + else + log ERROR "Failed to create GRANT files for ${db}" + return 1 + fi + done + + #Dumping the database + + SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.${SCOPE} + + $MYSQLDUMP $MYSQL_BACKUP_MYSQLDUMP_OPTIONS "${MYSQL_DBNAMES[@]}" \ + > $TMP_DIR/${SQL_FILE}.sql 2>>$LOG_FILE + if [[ $? -eq 0 && -s $TMP_DIR/${SQL_FILE}.sql ]] + then + log INFO "Database(s) dumped successfully. (SCOPE = ${SCOPE})" + return 0 + else + log ERROR "Backup failed and need attention. (SCOPE = ${SCOPE})" + return 1 + fi +} + +# functions from mariadb-verifier chart + +get_time_delta_secs () { + second_delta=0 + input_date_second=$( date --date="$1" +%s ) + if [ -n "$input_date_second" ]; then + current_date=$( date +"%Y-%m-%dT%H:%M:%SZ" ) + current_date_second=$( date --date="$current_date" +%s ) + ((second_delta=current_date_second-input_date_second)) + if [ "$second_delta" -lt 0 ]; then + second_delta=0 + fi + fi + echo $second_delta +} + + +check_data_freshness () { + archive_file=$(basename "$1") + archive_date=$(echo "$archive_file" | cut -d'.' -f 4) + SCOPE=$2 + + if [[ "${SCOPE}" != "all" ]]; then + log "Data freshness check is skipped for individual database." + return 0 + fi + + log "Checking for data freshness in the backups..." + # Get some idea of which database.table has changed in the last 30m + # Excluding the system DBs and aqua_test_database + # + changed_tables=$(${MYSQL_LIVE} -e "select TABLE_SCHEMA,TABLE_NAME from \ +information_schema.tables where UPDATE_TIME >= SUBTIME(now(),'00:30:00') AND TABLE_SCHEMA \ +NOT IN('information_schema', 'mysql', 'performance_schema', 'sys', 'aqua_test_database');" | \ +awk '{print $1 "." $2}') + + if [ -n "${changed_tables}" ]; then + delta_secs=$(get_time_delta_secs "$archive_date") + age_offset={{ .Values.conf.backup.validateData.ageOffset }} + ((age_threshold=delta_secs+age_offset)) + + data_freshness=false + skipped_freshness=false + + for table in ${changed_tables}; do + tab_schema=$(echo "$table" | awk -F. '{print $1}') + tab_name=$(echo "$table" | awk -F. '{print $2}') + + local_table_existed=$(${MYSQL_LOCAL_SHORT_SILENT} -e "select TABLE_SCHEMA,TABLE_NAME from \ +INFORMATION_SCHEMA.TABLES where TABLE_SCHEMA=\"${tab_schema}\" AND TABLE_NAME=\"${tab_name}\";") + + if [ -n "$local_table_existed" ]; then + # TODO: If last updated field of a table structure has different + # patterns (updated/timstamp), it may be worth to parameterize the patterns. + datetime=$(${MYSQL_LOCAL_SHORT_SILENT} -e "describe ${table};" | \ + awk '(/updated/ || /timestamp/) && /datetime/ {print $1}') + + if [ -n "${datetime}" ]; then + data_ages=$(${MYSQL_LOCAL_SHORT_SILENT} -e "select \ +time_to_sec(timediff(now(),${datetime})) from ${table} where ${datetime} is not null order by 1 limit 10;") + + for age in $data_ages; do + if [ "$age" -le $age_threshold ]; then + data_freshness=true + break + fi + done + + # As long as there is an indication of data freshness, no need to check further + if [ "$data_freshness" = true ] ; then + break + fi + else + skipped_freshness=true + log "No indicator to determine data freshness for table $table. Skipped data freshness check." + + # Dumping out table structure to determine if enhancement is needed to include this table + debug_info=$(${MYSQL_LOCAL} --skip-column-names -e "describe ${table};" | awk '{print $2 " " $1}') + log "$debug_info" "DEBUG" + fi + else + log "Table $table doesn't exist in local database" + skipped_freshness=true + fi + done + + if [ "$data_freshness" = true ] ; then + log "Database passed integrity (data freshness) check." + else + if [ "$skipped_freshness" = false ] ; then + log "Local backup database restore failed integrity check." "ERROR" + log "The backup may not have captured the up-to-date data." "INFO" + return 1 + fi + fi + else + log "No tables changed in this backup. Skipped data freshness check as the" + log "check should have been performed by previous validation runs." + fi + + return 0 +} + + +cleanup_local_databases () { + old_local_dbs=$(${MYSQL_LOCAL_SHORT_SILENT} -e 'show databases;' | \ + grep -ivE 'information_schema|performance_schema|mysql|sys' || true) + + for db in $old_local_dbs; do + ${MYSQL_LOCAL_SHORT_SILENT} -e "drop database $db;" + done +} + +list_archive_dir () { + archive_dir_content=$(ls -1R "$ARCHIVE_DIR") + if [ -n "$archive_dir_content" ]; then + log "Content of $ARCHIVE_DIR" + log "${archive_dir_content}" + fi +} + +remove_remote_archive_file () { + archive_file=$(basename "$1") + token_req_file=$(mktemp --suffix ".json") + header_file=$(mktemp) + resp_file=$(mktemp --suffix ".json") + http_resp="404" + + HEADER_CONTENT_TYPE="Content-Type: application/json" + HEADER_ACCEPT="Accept: application/json" + + cat << JSON_EOF > "$token_req_file" +{ + "auth": { + "identity": { + "methods": [ + "password" + ], + "password": { + "user": { + "domain": { + "name": "${OS_USER_DOMAIN_NAME}" + }, + "name": "${OS_USERNAME}", + "password": "${OS_PASSWORD}" + } + } + }, + "scope": { + "project": { + "domain": { + "name": "${OS_PROJECT_DOMAIN_NAME}" + }, + "name": "${OS_PROJECT_NAME}" + } + } + } +} +JSON_EOF + + http_resp=$(curl -s -X POST "$OS_AUTH_URL/auth/tokens" -H "${HEADER_CONTENT_TYPE}" \ + -H "${HEADER_ACCEPT}" -d @"${token_req_file}" -D "$header_file" -o "$resp_file" -w "%{http_code}") + + if [ "$http_resp" = "201" ]; then + OS_TOKEN=$(grep -i "x-subject-token" "$header_file" | cut -d' ' -f2 | tr -d "\r") + + if [ -n "$OS_TOKEN" ]; then + OS_OBJ_URL=$(python3 -c "import json,sys;print([[ep['url'] for ep in obj['endpoints'] if ep['interface']=='public'] for obj in json.load(sys.stdin)['token']['catalog'] if obj['type']=='object-store'][0][0])" < "$resp_file") + + if [ -n "$OS_OBJ_URL" ]; then + http_resp=$(curl -s -X DELETE "$OS_OBJ_URL/$CONTAINER_NAME/$archive_file" \ + -H "${HEADER_CONTENT_TYPE}" -H "${HEADER_ACCEPT}" \ + -H "X-Auth-Token: ${OS_TOKEN}" -D "$header_file" -o "$resp_file" -w "%{http_code}") + fi + fi + fi + + if [ "$http_resp" == "404" ] ; then + log "Failed to cleanup remote backup. Container object $archive_file is not on RGW." + return 1 + fi + + if [ "$http_resp" != "204" ] ; then + log "Failed to cleanup remote backup. Cannot delete container object $archive_file" "ERROR" + cat "$header_file" + cat "$resp_file" + fi + return 0 +} + +handle_bad_archive_file () { + archive_file=$1 + + if [ ! -d "$BAD_ARCHIVE_DIR" ]; then + mkdir -p "$BAD_ARCHIVE_DIR" + fi + + # Move the file to quarantine directory such that + # file won't be used for restore in case of recovery + # + log "Moving $i to $BAD_ARCHIVE_DIR..." + mv "$i" "$BAD_ARCHIVE_DIR" + log "Removing $i from remote RGW..." + if remove_remote_archive_file "$i"; then + log "File $i has been successfully removed from RGW." + else + log "FIle $i cannot be removed form RGW." "ERROR" + return 1 + fi + + # Atmost only three bad files are kept. Deleting the oldest if + # number of files exceeded the threshold. + # + bad_files=$(find "$BAD_ARCHIVE_DIR" -name "*.tar.gz" 2>/dev/null | wc -l) + if [ "$bad_files" -gt 3 ]; then + ((bad_files=bad_files-3)) + delete_files=$(find "$BAD_ARCHIVE_DIR" -name "*.tar.gz" 2>/dev/null | sort | head --lines=$bad_files) + for b in $delete_files; do + log "Deleting $b..." + rm -f "${b}" + done + fi + return 0 +} + +cleanup_old_validation_result_file () { + clean_files=$(find "$ARCHIVE_DIR" -maxdepth 1 -name "*.passed" 2>/dev/null) + for d in $clean_files; do + archive_file=${d/.passed} + if [ ! -f "$archive_file" ]; then + log "Deleting $d as its associated archive file $archive_file nolonger existed." + rm -f "${d}" + fi + done +} + +validate_databases_backup () { + archive_file=$1 + SCOPE=${2:-"all"} + + restore_log='/tmp/restore_error.log' + tmp_dir=$(mktemp -d) + + rm -f $restore_log + cd "$tmp_dir" + log "Decompressing archive $archive_file..." + if ! tar zxvf - < "$archive_file" 1>/dev/null; then + log "Database restore from local backup failed. Archive decompression failed." "ERROR" + return 1 + fi + + db_list_file="$tmp_dir/db.list" + if [[ -e "$db_list_file" ]]; then + dbs=$(sort < "$db_list_file" | grep -ivE sys | tr '\n' ' ') + else + dbs=" " + fi + + sql_file="${tmp_dir}/mariadb.${MARIADB_POD_NAMESPACE}.${SCOPE}.sql" + + if [[ "${SCOPE}" == "all" ]]; then + grant_file="${tmp_dir}/grants.sql" + else + grant_file="${tmp_dir}/${SCOPE}_grant.sql" + fi + + if [[ -f $sql_file ]]; then + if $MYSQL_LOCAL < "$sql_file" 2>$restore_log; then + local_dbs=$(${MYSQL_LOCAL_SHORT_SILENT} -e 'show databases;' | \ + grep -ivE 'information_schema|performance_schema|mysql|sys' | sort | tr '\n' ' ') + + if [ "$dbs" = "$local_dbs" ]; then + log "Databases restored successful." + else + log "Database restore from local backup failed. Database mismatched between local backup and local server" "ERROR" + log "Databases restored on local server: $local_dbs" "DEBUG" + log "Databases in the local backup: $dbs" "DEBUG" + return 1 + fi + else + log "Database restore from local backup failed. $dbs" "ERROR" + cat $restore_log + return 1 + fi + + if [[ -f $grant_file ]]; then + if $MYSQL_LOCAL < "$grant_file" 2>$restore_log; then + if ! $MYSQL_LOCAL -e 'flush privileges;'; then + log "Database restore from local backup failed. Failed to flush privileges." "ERROR" + return 1 + fi + log "Databases permission restored successful." + else + log "Database restore from local backup failed. Databases permission failed to restore." "ERROR" + cat "$restore_log" + cat "$grant_file" + log "Local DBs: $local_dbs" "DEBUG" + return 1 + fi + else + log "Database restore from local backup failed. There is no permission file available" "ERROR" + return 1 + fi + + if ! check_data_freshness "$archive_file" ${SCOPE}; then + # Log has already generated during check data freshness + return 1 + fi + else + log "Database restore from local backup failed. There is no database file available to restore from" "ERROR" + return 1 + fi + + return 0 +} + +# end of functions form mariadb verifier chart + +# Verify all the databases backup archives +verify_databases_backup_archives() { + SCOPE=${1:-"all"} + + # verification code + export DB_NAME="mariadb" + export ARCHIVE_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${MARIADB_POD_NAMESPACE}/${DB_NAME}/archive + export BAD_ARCHIVE_DIR=${ARCHIVE_DIR}/quarantine + export MYSQL_OPTS="--silent --skip-column-names" + export MYSQL_LIVE="mysql ${MYSQL_OPTS}" + export MYSQL_LOCAL_OPTS="" + export MYSQL_LOCAL_SHORT="mysql ${MYSQL_LOCAL_OPTS} --connect-timeout 2" + export MYSQL_LOCAL_SHORT_SILENT="${MYSQL_LOCAL_SHORT} ${MYSQL_OPTS}" + export MYSQL_LOCAL="mysql ${MYSQL_LOCAL_OPTS} --connect-timeout 10" + + max_wait={{ .Values.conf.mariadb_server.setup_wait.iteration }} + duration={{ .Values.conf.mariadb_server.setup_wait.duration }} + counter=0 + dbisup=false + + log "Waiting for Mariadb backup verification server to start..." + + # During Mariadb init/startup process, a temporary server is startup + # and shutdown prior to starting up the normal server. + # To avoid prematurely determine server availability, lets snooze + # a bit to give time for the process to complete prior to issue + # mysql commands. + # + + + while [ $counter -lt $max_wait ]; do + if ! $MYSQL_LOCAL_SHORT -e 'select 1' > /dev/null 2>&1 ; then + sleep $duration + ((counter=counter+1)) + else + # Lets sleep for an additional duration just in case async + # init takes a bit more time to complete. + # + sleep $duration + dbisup=true + counter=$max_wait + fi + done + + if ! $dbisup; then + log "Mariadb backup verification server is not running" "ERROR" + return 1 + fi + + # During Mariadb init process, a test database will be briefly + # created and deleted. Adding to the exclusion list for some + # edge cases + # + clean_db=$(${MYSQL_LOCAL_SHORT_SILENT} -e 'show databases;' | \ + grep -ivE 'information_schema|performance_schema|mysql|test|sys' || true) + + if [[ -z "${clean_db// }" ]]; then + log "Clean Server is up and running" + else + cleanup_local_databases + log "Old databases found on the Mariadb backup verification server were cleaned." + clean_db=$(${MYSQL_LOCAL_SHORT_SILENT} -e 'show databases;' | \ + grep -ivE 'information_schema|performance_schema|mysql|test|sys' || true) + + if [[ -z "${clean_db// }" ]]; then + log "Clean Server is up and running" + else + log "Cannot clean old databases on verification server." "ERROR" + return 1 + fi + log "The server is ready for verification." + fi + + # Starting with 10.4.13, new definer mariadb.sys was added. However, mariadb.sys was deleted + # during init mariadb as it was not on the exclusion list. This corrupted the view of mysql.user. + # Insert the tuple back to avoid other similar issues with error i.e + # The user specified as a definer ('mariadb.sys'@'localhost') does not exist + # + # Before insert the tuple mentioned above, we should make sure that the MariaDB version is 10.4.+ + mariadb_version=$($MYSQL_LOCAL_SHORT -e "status" | grep -E '^Server\s+version:') + log "Current database ${mariadb_version}" + if [[ ! -z ${mariadb_version} && -z $(grep '10.2' <<< ${mariadb_version}}) ]]; then + if [[ -z $(grep 'mariadb.sys' <<< $($MYSQL_LOCAL_SHORT mysql -e "select * from global_priv where user='mariadb.sys'")) ]]; then + $MYSQL_LOCAL_SHORT -e "insert into mysql.global_priv values ('localhost','mariadb.sys',\ + '{\"access\":0,\"plugin\":\"mysql_native_password\",\"authentication_string\":\"\",\"account_locked\":true,\"password_last_changed\":0}');" + $MYSQL_LOCAL_SHORT -e 'flush privileges;' + fi + fi + + # Ensure archive dir existed + if [ -d "$ARCHIVE_DIR" ]; then + # List archive dir before + list_archive_dir + + # Ensure the local databases are clean for each restore validation + # + cleanup_local_databases + + if [[ "${SCOPE}" == "all" ]]; then + archive_files=$(find "$ARCHIVE_DIR" -maxdepth 1 -name "*.tar.gz" 2>/dev/null | sort) + for i in $archive_files; do + archive_file_passed=$i.passed + if [ ! -f "$archive_file_passed" ]; then + log "Validating archive file $i..." + if validate_databases_backup "$i"; then + touch "$archive_file_passed" + else + if handle_bad_archive_file "$i"; then + log "File $i has been removed from RGW." + else + log "File $i cannot be removed from RGW." "ERROR" + return 1 + fi + fi + fi + done + else + archive_files=$(find "$ARCHIVE_DIR" -maxdepth 1 -name "*.tar.gz" 2>/dev/null | grep "${SCOPE}" | sort) + for i in $archive_files; do + archive_file_passed=$i.passed + if [ ! -f "$archive_file_passed" ]; then + log "Validating archive file $i..." + if validate_databases_backup "${i}" "${SCOPE}"; then + touch "$archive_file_passed" + else + if handle_bad_archive_file "$i"; then + log "File $i has been removed from RGW." + else + log "File $i cannot be removed from RGW." "ERROR" + return 1 + fi + fi + fi + done + fi + + + # Cleanup passed files if its archive file nolonger existed + cleanup_old_validation_result_file + + # List archive dir after + list_archive_dir + fi + + + return 0 +} + +# Call main program to start the database backup +backup_databases ${SCOPE} diff --git a/mariadb-backup/templates/bin/_restore_mariadb.sh.tpl b/mariadb-backup/templates/bin/_restore_mariadb.sh.tpl new file mode 100755 index 0000000000..334ba85bc6 --- /dev/null +++ b/mariadb-backup/templates/bin/_restore_mariadb.sh.tpl @@ -0,0 +1,328 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +{{- $envAll := . }} + +# Capture the user's command line arguments +ARGS=("$@") + +if [[ -s /tmp/restore_main.sh ]]; then + source /tmp/restore_main.sh +else + echo "File /tmp/restore_main.sh does not exist." + exit 1 +fi + +# Export the variables needed by the framework +export DB_NAME="mariadb" +export DB_NAMESPACE=${MARIADB_POD_NAMESPACE} +export ARCHIVE_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${DB_NAMESPACE}/${DB_NAME}/archive + +RESTORE_USER='restoreuser' +RESTORE_PW=$(pwgen 16 1) +RESTORE_LOG='/tmp/restore_error.log' +rm -f $RESTORE_LOG + +# This is for commands which require admin access +MYSQL="mysql \ + --defaults-file=/etc/mysql/admin_user.cnf \ + --host=$MARIADB_SERVER_SERVICE_HOST \ + --connect-timeout 10" + +# This is for commands which we want the temporary "restore" user +# to execute +RESTORE_CMD="mysql \ + --user=${RESTORE_USER} \ + --password=${RESTORE_PW} \ + --host=$MARIADB_SERVER_SERVICE_HOST \ +{{- if .Values.manifests.certificates }} + --ssl-ca=/etc/mysql/certs/ca.crt \ + --ssl-key=/etc/mysql/certs/tls.key \ + --ssl-cert=/etc/mysql/certs/tls.crt \ +{{- end }} + --connect-timeout 10" + +# Get a single database data from the SQL file. +# $1 - database name +# $2 - sql file path +current_db_desc() { + PATTERN="-- Current Database:" + sed -n "/${PATTERN} \`$1\`/,/${PATTERN}/p" $2 +} + +#Return all database from an archive +get_databases() { + TMP_DIR=$1 + DB_FILE=$2 + + if [[ -e ${TMP_DIR}/db.list ]] + then + DBS=$(cat ${TMP_DIR}/db.list | \ + grep -ivE 'information_schema|performance_schema|mysql|sys' ) + else + DBS=" " + fi + + echo $DBS > $DB_FILE +} + +# Determine sql file from 2 options - current and legacy one +# if current is not found check that there is no other namespaced dump file +# before falling back to legacy one +_get_sql_file() { + TMP_DIR=$1 + SQL_FILE="${TMP_DIR}/mariadb.${MARIADB_POD_NAMESPACE}.*.sql" + LEGACY_SQL_FILE="${TMP_DIR}/mariadb.*.sql" + INVALID_SQL_FILE="${TMP_DIR}/mariadb.*.*.sql" + if [ -f ${SQL_FILE} ] + then + echo "Found $(ls ${SQL_FILE})" > /dev/stderr + printf ${SQL_FILE} + elif [ -f ${INVALID_SQL_FILE} ] + then + echo "Expected to find ${SQL_FILE} or ${LEGACY_SQL_FILE}, but found $(ls ${INVALID_SQL_FILE})" > /dev/stderr + elif [ -f ${LEGACY_SQL_FILE} ] + then + echo "Falling back to legacy naming ${LEGACY_SQL_FILE}. Found $(ls ${LEGACY_SQL_FILE})" > /dev/stderr + printf ${LEGACY_SQL_FILE} + fi +} + +# Extract all tables of a database from an archive and put them in the requested +# file. +get_tables() { + DATABASE=$1 + TMP_DIR=$2 + TABLE_FILE=$3 + + SQL_FILE=$(_get_sql_file $TMP_DIR) + if [ ! -z $SQL_FILE ]; then + current_db_desc ${DATABASE} ${SQL_FILE} \ + | grep "^CREATE TABLE" | awk -F '`' '{print $2}' \ + > $TABLE_FILE + else + # Error, cannot report the tables + echo "No SQL file found - cannot extract the tables" + return 1 + fi +} + +# Extract all rows in the given table of a database from an archive and put +# them in the requested file. +get_rows() { + DATABASE=$1 + TABLE=$2 + TMP_DIR=$3 + ROW_FILE=$4 + + SQL_FILE=$(_get_sql_file $TMP_DIR) + if [ ! -z $SQL_FILE ]; then + current_db_desc ${DATABASE} ${SQL_FILE} \ + | grep "INSERT INTO \`${TABLE}\` VALUES" > $ROW_FILE + return 0 + else + # Error, cannot report the rows + echo "No SQL file found - cannot extract the rows" + return 1 + fi +} + +# Extract the schema for the given table in the given database belonging to +# the archive file found in the TMP_DIR. +get_schema() { + DATABASE=$1 + TABLE=$2 + TMP_DIR=$3 + SCHEMA_FILE=$4 + + SQL_FILE=$(_get_sql_file $TMP_DIR) + if [ ! -z $SQL_FILE ]; then + DB_FILE=$(mktemp -p /tmp) + current_db_desc ${DATABASE} ${SQL_FILE} > ${DB_FILE} + sed -n /'CREATE TABLE `'$TABLE'`'/,/'--'/p ${DB_FILE} > ${SCHEMA_FILE} + if [[ ! (-s ${SCHEMA_FILE}) ]]; then + sed -n /'CREATE TABLE IF NOT EXISTS `'$TABLE'`'/,/'--'/p ${DB_FILE} \ + > ${SCHEMA_FILE} + fi + rm -f ${DB_FILE} + else + # Error, cannot report the rows + echo "No SQL file found - cannot extract the schema" + return 1 + fi +} + +# Create temporary user for restoring specific databases. +create_restore_user() { + restore_db=$1 + + # Ensure any old restore user is removed first, if it exists. + # If it doesn't exist it may return error, so do not exit the + # script if that's the case. + delete_restore_user "dont_exit_on_error" + + $MYSQL --execute="GRANT SELECT ON *.* TO ${RESTORE_USER}@'%' IDENTIFIED BY '${RESTORE_PW}';" 2>>$RESTORE_LOG + if [[ "$?" -eq 0 ]] + then + $MYSQL --execute="GRANT ALL ON ${restore_db}.* TO ${RESTORE_USER}@'%' IDENTIFIED BY '${RESTORE_PW}';" 2>>$RESTORE_LOG + if [[ "$?" -ne 0 ]] + then + cat $RESTORE_LOG + echo "Failed to grant restore user ALL permissions on database ${restore_db}" + return 1 + fi + else + cat $RESTORE_LOG + echo "Failed to grant restore user select permissions on all databases" + return 1 + fi +} + +# Delete temporary restore user +delete_restore_user() { + error_handling=$1 + + $MYSQL --execute="DROP USER ${RESTORE_USER}@'%';" 2>>$RESTORE_LOG + if [[ "$?" -ne 0 ]] + then + if [ "$error_handling" == "exit_on_error" ] + then + cat $RESTORE_LOG + echo "Failed to delete temporary restore user - needs attention to avoid a security hole" + return 1 + fi + fi +} + +#Restore a single database +restore_single_db() { + SINGLE_DB_NAME=$1 + TMP_DIR=$2 + + if [[ -z "$SINGLE_DB_NAME" ]] + then + echo "Restore single DB called but with wrong parameter." + return 1 + fi + + SQL_FILE=$(_get_sql_file $TMP_DIR) + if [ ! -z $SQL_FILE ]; then + # Restoring a single database requires us to create a temporary user + # which has capability to only restore that ONE database. One gotcha + # is that the mysql command to restore the database is going to throw + # errors because of all the other databases that it cannot access. So + # because of this reason, the --force option is used to prevent the + # command from stopping on an error. + create_restore_user $SINGLE_DB_NAME + if [[ $? -ne 0 ]] + then + echo "Restore $SINGLE_DB_NAME failed create restore user." + return 1 + fi + $RESTORE_CMD --force < $SQL_FILE 2>>$RESTORE_LOG + if [[ "$?" -eq 0 ]] + then + echo "Database $SINGLE_DB_NAME Restore successful." + else + cat $RESTORE_LOG + delete_restore_user "exit_on_error" + echo "Database $SINGLE_DB_NAME Restore failed." + return 1 + fi + delete_restore_user "exit_on_error" + if [[ $? -ne 0 ]] + then + echo "Restore $SINGLE_DB_NAME failed delete restore user." + return 1 + fi + if [ -f ${TMP_DIR}/${SINGLE_DB_NAME}_grant.sql ] + then + $MYSQL < ${TMP_DIR}/${SINGLE_DB_NAME}_grant.sql 2>>$RESTORE_LOG + if [[ "$?" -eq 0 ]] + then + if ! $MYSQL --execute="FLUSH PRIVILEGES;"; then + echo "Failed to flush privileges for $SINGLE_DB_NAME." + return 1 + fi + echo "Database $SINGLE_DB_NAME Permission Restore successful." + else + cat $RESTORE_LOG + echo "Database $SINGLE_DB_NAME Permission Restore failed." + return 1 + fi + else + echo "There is no permission file available for $SINGLE_DB_NAME" + return 1 + fi + else + echo "There is no database file available to restore from" + return 1 + fi + return 0 +} + +#Restore all the databases +restore_all_dbs() { + TMP_DIR=$1 + + SQL_FILE=$(_get_sql_file $TMP_DIR) + if [ ! -z $SQL_FILE ]; then + # Check the scope of the archive. + SCOPE=$(echo ${SQL_FILE} | awk -F'.' '{print $(NF-1)}') + if [[ "${SCOPE}" != "all" ]]; then + # This is just a single database backup. The user should + # instead use the single database restore option. + echo "Cannot use the restore all option for an archive containing only a single database." + echo "Please use the single database restore option." + return 1 + fi + + $MYSQL < $SQL_FILE 2>$RESTORE_LOG + if [[ "$?" -eq 0 ]] + then + echo "Databases $( echo $DBS | tr -d '\n') Restore successful." + else + cat $RESTORE_LOG + echo "Databases $( echo $DBS | tr -d '\n') Restore failed." + return 1 + fi + if [[ -f ${TMP_DIR}/grants.sql ]] + then + $MYSQL < ${TMP_DIR}/grants.sql 2>$RESTORE_LOG + if [[ "$?" -eq 0 ]] + then + if ! $MYSQL --execute="FLUSH PRIVILEGES;"; then + echo "Failed to flush privileges." + return 1 + fi + echo "Databases Permission Restore successful." + else + cat $RESTORE_LOG + echo "Databases Permission Restore failed." + return 1 + fi + else + echo "There is no permission file available" + return 1 + fi + else + echo "There is no database file available to restore from" + return 1 + fi + return 0 +} + +# Call the CLI interpreter, providing the archive directory path and the +# user arguments passed in +cli_main ${ARGS[@]} diff --git a/mariadb-backup/templates/bin/_start_mariadb_verify_server.sh.tpl b/mariadb-backup/templates/bin/_start_mariadb_verify_server.sh.tpl new file mode 100644 index 0000000000..c633946c93 --- /dev/null +++ b/mariadb-backup/templates/bin/_start_mariadb_verify_server.sh.tpl @@ -0,0 +1,29 @@ +#!/bin/bash -ex + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +log () { + msg_default="Need some text to log" + level_default="INFO" + component_default="Mariadb Backup Verifier" + + msg=${1:-$msg_default} + level=${2:-$level_default} + component=${3:-"$component_default"} + + echo "$(date +'%Y-%m-%d %H:%M:%S,%3N') - ${component} - ${level} - ${msg}" +} + +log "Starting Mariadb server for backup verification..." +mysql_install_db --user=nobody --ldata=/var/lib/mysql >/dev/null 2>&1 +MYSQL_ALLOW_EMPTY_PASSWORD=1 mysqld --user=nobody --verbose >/dev/null 2>&1 diff --git a/mariadb-backup/templates/configmap-bin.yaml b/mariadb-backup/templates/configmap-bin.yaml new file mode 100644 index 0000000000..2c8b1cc5b4 --- /dev/null +++ b/mariadb-backup/templates/configmap-bin.yaml @@ -0,0 +1,45 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_bin }} +{{- $envAll := . }} +{{ if eq .Values.endpoints.oslo_db.auth.admin.username .Values.endpoints.oslo_db.auth.sst.username }} +{{ fail "the DB admin username should not match the sst user username" }} +{{ end }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: mariadb-backup-bin +data: + backup_mariadb.sh: | +{{ tuple "bin/_backup_mariadb.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + start_verification_server.sh: | +{{ tuple "bin/_start_mariadb_verify_server.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + restore_mariadb.sh: | +{{ tuple "bin/_restore_mariadb.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + backup_main.sh: | +{{ include "helm-toolkit.scripts.db-backup-restore.backup_main" . | indent 4 }} + restore_main.sh: | +{{ include "helm-toolkit.scripts.db-backup-restore.restore_main" . | indent 4 }} +{{- if .Values.images.local_registry.active }} + image-repo-sync.sh: | +{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} +{{- end }} +{{- if .Values.manifests.job_ks_user }} + ks-user.sh: | +{{ include "helm-toolkit.scripts.keystone_user" . | indent 4 }} +{{- end }} +{{- end }} +... diff --git a/mariadb-backup/templates/configmap-etc.yaml b/mariadb-backup/templates/configmap-etc.yaml new file mode 100644 index 0000000000..1f792ab389 --- /dev/null +++ b/mariadb-backup/templates/configmap-etc.yaml @@ -0,0 +1,24 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License" ); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_etc }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: mariadb-backup-etc +data: +{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" ( index $envAll.Values.conf.database "my" ) "key" "my.cnf" ) | indent 2 }} +{{- end }} diff --git a/mariadb-backup/templates/cron-job-backup-mariadb.yaml b/mariadb-backup/templates/cron-job-backup-mariadb.yaml new file mode 100644 index 0000000000..d84ec16942 --- /dev/null +++ b/mariadb-backup/templates/cron-job-backup-mariadb.yaml @@ -0,0 +1,210 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.cron_job_mariadb_backup }} +{{- $envAll := . }} + +{{- $serviceAccountName := "mariadb-backup" }} +{{ tuple $envAll "mariadb_backup" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: mariadb-backup + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "mariadb-backup" "backup" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + schedule: {{ .Values.jobs.mariadb_backup.cron | quote }} + successfulJobsHistoryLimit: {{ .Values.jobs.mariadb_backup.history.success }} + failedJobsHistoryLimit: {{ .Values.jobs.mariadb_backup.history.failed }} + concurrencyPolicy: Forbid + jobTemplate: + metadata: + labels: +{{ tuple $envAll "mariadb-backup" "backup" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ dict "envAll" $envAll "podName" "mariadb-backup" "containerNames" (list "init" "backup-perms" "mariadb-backup") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{- if .Values.jobs.mariadb_backup.backoffLimit }} + backoffLimit: {{ .Values.jobs.mariadb_backup.backoffLimit }} +{{- end }} +{{- if .Values.jobs.mariadb_backup.activeDeadlineSeconds }} + activeDeadlineSeconds: {{ .Values.jobs.mariadb_backup.activeDeadlineSeconds }} +{{- end }} + template: + metadata: + labels: +{{ tuple $envAll "mariadb-backup" "backup" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 12 }} + spec: +{{ dict "envAll" $envAll "application" "mariadb_backup" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 10 }} + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + shareProcessNamespace: true +{{- if $envAll.Values.pod.tolerations.mariadb.enabled }} +{{ tuple $envAll "mariadb" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 10 }} +{{- end }} +{{- if $envAll.Values.pod.affinity }} +{{- if $envAll.Values.pod.affinity.mariadb_backup }} + affinity: +{{ index $envAll.Values.pod.affinity "mariadb_backup" | toYaml | indent 12}} +{{- end }} +{{- end }} + nodeSelector: + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} + initContainers: +{{ tuple $envAll "mariadb_backup" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 12 }} + - name: backup-perms +{{ tuple $envAll "mariadb_backup" | include "helm-toolkit.snippets.image" | indent 14 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.mariadb_backup | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }} +{{ dict "envAll" $envAll "application" "mariadb_backup" "container" "backup_perms" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 14 }} + command: + - chown + - -R + - "65534:65534" + - $(MARIADB_BACKUP_BASE_DIR) + env: + - name: MARIADB_BACKUP_BASE_DIR + value: {{ .Values.conf.backup.base_path | quote }} + volumeMounts: + - mountPath: /tmp + name: pod-tmp + - mountPath: {{ .Values.conf.backup.base_path }} + name: mariadb-backup-dir + - name: verify-perms +{{ tuple $envAll "mariadb_backup" | include "helm-toolkit.snippets.image" | indent 14 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.mariadb_backup | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }} +{{ dict "envAll" $envAll "application" "mariadb_backup" "container" "verify_perms" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 14 }} + command: + - chown + - -R + - "65534:65534" + - /var/lib/mysql + volumeMounts: + - mountPath: /tmp + name: pod-tmp + - mountPath: /var/lib/mysql + name: mysql-data + containers: + - name: mariadb-backup + command: + - /bin/sh + args: + - -c + - >- + ( /tmp/start_verification_server.sh ) & + /tmp/backup_mariadb.sh + env: + - name: MARIADB_BACKUP_BASE_DIR + value: {{ .Values.conf.backup.base_path | quote }} + - name: MYSQL_BACKUP_MYSQLDUMP_OPTIONS + value: {{ .Values.conf.backup.mysqldump_options | quote }} + - name: MARIADB_LOCAL_BACKUP_DAYS_TO_KEEP + value: {{ .Values.conf.backup.days_to_keep | quote }} + - name: MARIADB_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: REMOTE_BACKUP_ENABLED + value: "{{ .Values.conf.backup.remote_backup.enabled }}" +{{- if .Values.conf.backup.remote_backup.enabled }} + - name: MARIADB_REMOTE_BACKUP_DAYS_TO_KEEP + value: {{ .Values.conf.backup.remote_backup.days_to_keep | quote }} + - name: CONTAINER_NAME + value: {{ .Values.conf.backup.remote_backup.container_name | quote }} + - name: STORAGE_POLICY + value: "{{ .Values.conf.backup.remote_backup.storage_policy }}" + - name: NUMBER_OF_RETRIES_SEND_BACKUP_TO_REMOTE + value: {{ .Values.conf.backup.remote_backup.number_of_retries | quote }} + - name: MIN_DELAY_SEND_BACKUP_TO_REMOTE + value: {{ .Values.conf.backup.remote_backup.delay_range.min | quote }} + - name: MAX_DELAY_SEND_BACKUP_TO_REMOTE + value: {{ .Values.conf.backup.remote_backup.delay_range.max | quote }} + - name: THROTTLE_BACKUPS_ENABLED + value: "{{ .Values.conf.backup.remote_backup.throttle_backups.enabled }}" + - name: THROTTLE_LIMIT + value: {{ .Values.conf.backup.remote_backup.throttle_backups.sessions_limit | quote }} + - name: THROTTLE_LOCK_EXPIRE_AFTER + value: {{ .Values.conf.backup.remote_backup.throttle_backups.lock_expire_after | quote }} + - name: THROTTLE_RETRY_AFTER + value: {{ .Values.conf.backup.remote_backup.throttle_backups.retry_after | quote }} + - name: THROTTLE_CONTAINER_NAME + value: {{ .Values.conf.backup.remote_backup.throttle_backups.container_name | quote }} +{{- with $env := dict "ksUserSecret" $envAll.Values.secrets.identity.mariadb }} +{{- include "helm-toolkit.snippets.keystone_openrc_env_vars" $env | indent 16 }} +{{- end }} +{{- end }} +{{ tuple $envAll "mariadb_backup" | include "helm-toolkit.snippets.image" | indent 14 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.mariadb_backup | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }} +{{ dict "envAll" $envAll "application" "mariadb_backup" "container" "mariadb_backup" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 14 }} + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - mountPath: /tmp/backup_mariadb.sh + name: mariadb-backup-bin + readOnly: true + subPath: backup_mariadb.sh + - mountPath: /tmp/backup_main.sh + name: mariadb-backup-bin + readOnly: true + subPath: backup_main.sh + - mountPath: {{ .Values.conf.backup.base_path }} + name: mariadb-backup-dir + - name: mariadb-backup-secrets + mountPath: /etc/mysql/admin_user.cnf + subPath: admin_user.cnf + readOnly: true + - name: mariadb-backup-bin + mountPath: /tmp/start_verification_server.sh + readOnly: true + subPath: start_verification_server.sh + - name: mysql-data + mountPath: /var/lib/mysql + - name: var-run + mountPath: /run/mysqld +{{ dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.secrets.tls.oslo_db.server.internal "path" "/etc/mysql/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 16 }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: mycnfd + emptyDir: {} + - name: var-run + emptyDir: {} + - name: mariadb-backup-etc + configMap: + name: mariadb-backup-etc + defaultMode: 0444 + - name: mysql-data + emptyDir: {} + - name: mariadb-backup-secrets + secret: + secretName: mariadb-backup-secrets + defaultMode: 420 + - configMap: + defaultMode: 365 + name: mariadb-backup-bin + name: mariadb-backup-bin + {{- if and .Values.volume.backup.enabled .Values.manifests.pvc_backup }} + - name: mariadb-backup-dir + persistentVolumeClaim: + claimName: mariadb-backup-data + {{- else }} + - hostPath: + path: {{ .Values.conf.backup.base_path }} + type: DirectoryOrCreate + name: mariadb-backup-dir + {{- end }} +{{ dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.secrets.tls.oslo_db.server.internal | include "helm-toolkit.snippets.tls_volume" | indent 12 }} +{{- end }} diff --git a/mariadb-backup/templates/job-image-repo-sync.yaml b/mariadb-backup/templates/job-image-repo-sync.yaml new file mode 100644 index 0000000000..2f59221ad7 --- /dev/null +++ b/mariadb-backup/templates/job-image-repo-sync.yaml @@ -0,0 +1,22 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $serviceName := tuple "oslo_db" "server" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" $serviceName -}} +{{- if .Values.pod.tolerations.mariadb.enabled -}} +{{- $_ := set $imageRepoSyncJob "tolerationsEnabled" true -}} +{{- end -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/mariadb-backup/templates/job-ks-user.yaml b/mariadb-backup/templates/job-ks-user.yaml new file mode 100644 index 0000000000..bc7befa389 --- /dev/null +++ b/mariadb-backup/templates/job-ks-user.yaml @@ -0,0 +1,24 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.job_ks_user }} +{{- $backoffLimit := .Values.jobs.ks_user.backoffLimit }} +{{- $activeDeadlineSeconds := .Values.jobs.ks_user.activeDeadlineSeconds }} +{{- $serviceName := tuple "oslo_db" "server" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +{{- $ksUserJob := dict "envAll" . "serviceName" $serviceName "configMapBin" "mariadb-backup-bin" "backoffLimit" $backoffLimit "activeDeadlineSeconds" $activeDeadlineSeconds -}} +{{- if .Values.pod.tolerations.mariadb.enabled -}} +{{- $_ := set $ksUserJob "tolerationsEnabled" true -}} +{{- end -}} +{{ $ksUserJob | include "helm-toolkit.manifests.job_ks_user" }} +{{- end }} diff --git a/mariadb-backup/templates/mariadb-backup-pvc.yaml b/mariadb-backup/templates/mariadb-backup-pvc.yaml new file mode 100644 index 0000000000..e2b5827651 --- /dev/null +++ b/mariadb-backup/templates/mariadb-backup-pvc.yaml @@ -0,0 +1,29 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.volume.backup.enabled .Values.manifests.pvc_backup }} +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: mariadb-backup-data +spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: {{ .Values.volume.backup.size }} + storageClassName: {{ .Values.volume.backup.class_name }} +... +{{- end }} + diff --git a/mariadb-backup/templates/secret-backup-restore.yaml b/mariadb-backup/templates/secret-backup-restore.yaml new file mode 100644 index 0000000000..1a37290b70 --- /dev/null +++ b/mariadb-backup/templates/secret-backup-restore.yaml @@ -0,0 +1,35 @@ +{{/* +This manifest results a secret being created which has the key information +needed for backing up and restoring the Mariadb databases. +*/}} + +{{- if and .Values.conf.backup.enabled .Values.manifests.secret_backup_restore }} + +{{- $envAll := . }} +{{- $userClass := "backup_restore" }} +{{- $secretName := index $envAll.Values.secrets.mariadb $userClass }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} +type: Opaque +data: + BACKUP_ENABLED: {{ $envAll.Values.conf.backup.enabled | quote | b64enc }} + BACKUP_BASE_PATH: {{ $envAll.Values.conf.backup.base_path | b64enc }} + LOCAL_DAYS_TO_KEEP: {{ $envAll.Values.conf.backup.days_to_keep | quote | b64enc }} + MYSQLDUMP_OPTIONS: {{ $envAll.Values.conf.backup.mysqldump_options | b64enc }} + REMOTE_BACKUP_ENABLED: {{ $envAll.Values.conf.backup.remote_backup.enabled | quote | b64enc }} + REMOTE_BACKUP_CONTAINER: {{ $envAll.Values.conf.backup.remote_backup.container_name | b64enc }} + REMOTE_BACKUP_DAYS_TO_KEEP: {{ $envAll.Values.conf.backup.remote_backup.days_to_keep | quote | b64enc }} + REMOTE_BACKUP_STORAGE_POLICY: {{ $envAll.Values.conf.backup.remote_backup.storage_policy | b64enc }} + REMOTE_BACKUP_RETRIES: {{ $envAll.Values.conf.backup.remote_backup.number_of_retries | quote | b64enc }} + REMOTE_BACKUP_SEND_DELAY_MIN: {{ $envAll.Values.conf.backup.remote_backup.delay_range.min | quote | b64enc }} + REMOTE_BACKUP_SEND_DELAY_MAX: {{ $envAll.Values.conf.backup.remote_backup.delay_range.max | quote | b64enc }} + THROTTLE_BACKUPS_ENABLED: {{ $envAll.Values.conf.backup.remote_backup.throttle_backups.enabled | quote | b64enc }} + THROTTLE_LIMIT: {{ $envAll.Values.conf.backup.remote_backup.throttle_backups.sessions_limit | quote | b64enc }} + THROTTLE_LOCK_EXPIRE_AFTER: {{ $envAll.Values.conf.backup.remote_backup.throttle_backups.lock_expire_after | quote | b64enc }} + THROTTLE_RETRY_AFTER: {{ $envAll.Values.conf.backup.remote_backup.throttle_backups.retry_after | quote | b64enc }} + THROTTLE_CONTAINER_NAME: {{ $envAll.Values.conf.backup.remote_backup.throttle_backups.container_name | quote | b64enc }} +... +{{- end }} diff --git a/mariadb-backup/templates/secret-registry.yaml b/mariadb-backup/templates/secret-registry.yaml new file mode 100644 index 0000000000..da979b3223 --- /dev/null +++ b/mariadb-backup/templates/secret-registry.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_registry .Values.endpoints.oci_image_registry.auth.enabled }} +{{ include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) }} +{{- end }} diff --git a/mariadb-backup/templates/secret-rgw.yaml b/mariadb-backup/templates/secret-rgw.yaml new file mode 100644 index 0000000000..bdb9ca098b --- /dev/null +++ b/mariadb-backup/templates/secret-rgw.yaml @@ -0,0 +1,78 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +This manifest results in two secrets being created: + 1) Keystone "mariadb" secret, which is needed to access the cluster + (remote or same cluster) for storing mariadb backups. If the + cluster is remote, the auth_url would be non-null. + 2) Keystone "admin" secret, which is needed to create the + "mariadb" keystone account mentioned above. This may not + be needed if the account is in a remote cluster (auth_url is non-null + in that case). +*/}} + +{{- if .Values.conf.backup.remote_backup.enabled }} + +{{- $envAll := . }} +{{- $userClass := "mariadb-server" }} +{{- $secretName := index $envAll.Values.secrets.identity $userClass }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} +type: Opaque +data: +{{- $identityClass := index .Values.endpoints.identity.auth $userClass }} +{{- if $identityClass.auth_url }} + OS_AUTH_URL: {{ $identityClass.auth_url | b64enc }} +{{- else }} + OS_AUTH_URL: {{ tuple "identity" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | b64enc }} +{{- end }} + OS_REGION_NAME: {{ $identityClass.region_name | b64enc }} + OS_INTERFACE: {{ $identityClass.interface | default "internal" | b64enc }} + OS_PROJECT_DOMAIN_NAME: {{ $identityClass.project_domain_name | b64enc }} + OS_PROJECT_NAME: {{ $identityClass.project_name | b64enc }} + OS_USER_DOMAIN_NAME: {{ $identityClass.user_domain_name | b64enc }} + OS_USERNAME: {{ $identityClass.username | b64enc }} + OS_PASSWORD: {{ $identityClass.password | b64enc }} + OS_DEFAULT_DOMAIN: {{ $identityClass.default_domain_id | default "default" | b64enc }} +... +{{- if .Values.manifests.job_ks_user }} +{{- $userClass := "admin" }} +{{- $secretName := index $envAll.Values.secrets.identity $userClass }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} +type: Opaque +data: +{{- $identityClass := index .Values.endpoints.identity.auth $userClass }} +{{- if $identityClass.auth_url }} + OS_AUTH_URL: {{ $identityClass.auth_url | b64enc }} +{{- else }} + OS_AUTH_URL: {{ tuple "identity" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | b64enc }} +{{- end }} + OS_REGION_NAME: {{ $identityClass.region_name | b64enc }} + OS_INTERFACE: {{ $identityClass.interface | default "internal" | b64enc }} + OS_PROJECT_DOMAIN_NAME: {{ $identityClass.project_domain_name | b64enc }} + OS_PROJECT_NAME: {{ $identityClass.project_name | b64enc }} + OS_USER_DOMAIN_NAME: {{ $identityClass.user_domain_name | b64enc }} + OS_USERNAME: {{ $identityClass.username | b64enc }} + OS_PASSWORD: {{ $identityClass.password | b64enc }} + OS_DEFAULT_DOMAIN: {{ $identityClass.default_domain_id | default "default" | b64enc }} +... +{{- end }} +{{- end }} diff --git a/mariadb-backup/templates/secrets-etc.yaml b/mariadb-backup/templates/secrets-etc.yaml new file mode 100644 index 0000000000..de29258479 --- /dev/null +++ b/mariadb-backup/templates/secrets-etc.yaml @@ -0,0 +1,26 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_etc }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: mariadb-backup-secrets +type: Opaque +data: + admin_user.cnf: {{ tuple "secrets/_admin_user.cnf.tpl" . | include "helm-toolkit.utils.template" | b64enc }} + admin_user_internal.cnf: {{ tuple "secrets/_admin_user_internal.cnf.tpl" . | include "helm-toolkit.utils.template" | b64enc }} +{{- end }} diff --git a/mariadb-backup/templates/secrets/_admin_user.cnf.tpl b/mariadb-backup/templates/secrets/_admin_user.cnf.tpl new file mode 100644 index 0000000000..0031a4bd7d --- /dev/null +++ b/mariadb-backup/templates/secrets/_admin_user.cnf.tpl @@ -0,0 +1,24 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +[client] +user = {{ .Values.endpoints.oslo_db.auth.admin.username }} +password = {{ .Values.endpoints.oslo_db.auth.admin.password }} +host = {{ tuple "oslo_db" "direct" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +port = {{ tuple "oslo_db" "direct" "mysql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +{{- if .Values.manifests.certificates }} +ssl-ca = /etc/mysql/certs/ca.crt +ssl-key = /etc/mysql/certs/tls.key +ssl-cert = /etc/mysql/certs/tls.crt +{{- end }} diff --git a/mariadb-backup/templates/secrets/_admin_user_internal.cnf.tpl b/mariadb-backup/templates/secrets/_admin_user_internal.cnf.tpl new file mode 100644 index 0000000000..fa0d09a559 --- /dev/null +++ b/mariadb-backup/templates/secrets/_admin_user_internal.cnf.tpl @@ -0,0 +1,24 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +[client] +user = {{ .Values.endpoints.oslo_db.auth.admin.username }} +password = {{ .Values.endpoints.oslo_db.auth.admin.password }} +host = {{ tuple "oslo_db" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }} +port = {{ tuple "oslo_db" "internal" "mysql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +{{- if .Values.manifests.certificates }} +ssl-ca = /etc/mysql/certs/ca.crt +ssl-key = /etc/mysql/certs/tls.key +ssl-cert = /etc/mysql/certs/tls.crt +{{- end }} diff --git a/mariadb-backup/values.yaml b/mariadb-backup/values.yaml new file mode 100644 index 0000000000..2770f63863 --- /dev/null +++ b/mariadb-backup/values.yaml @@ -0,0 +1,385 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for mariadb. +# This is a YAML-formatted file. +# Declare name/value pairs to be passed into your templates. +# name: value + +--- +release_group: null + +images: + tags: + mariadb: docker.io/openstackhelm/mariadb:latest-ubuntu_focal + ks_user: docker.io/openstackhelm/heat:wallaby-ubuntu_focal + dep_check: quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal + mariadb_backup: quay.io/airshipit/porthole-mysqlclient-utility:latest-ubuntu_focal + pull_policy: "IfNotPresent" + local_registry: + active: false + exclude: + - dep_check + +labels: + server: + node_selector_key: openstack-control-plane + node_selector_value: enabled + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +pod: + security_context: + server: + pod: + runAsUser: 999 + container: + perms: + runAsUser: 0 + readOnlyRootFilesystem: true + init: + runAsUser: 0 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false + agent: + runAsUser: 0 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false + mariadb: + runAsUser: 0 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false + mariadb_backup: + pod: + runAsUser: 65534 + container: + backup_perms: + runAsUser: 0 + readOnlyRootFilesystem: true + verify_perms: + runAsUser: 0 + readOnlyRootFilesystem: true + mariadb_backup: + runAsUser: 65534 + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + tests: + pod: + runAsUser: 999 + container: + test: + runAsUser: 999 + readOnlyRootFilesystem: true + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + weight: + default: 10 + tolerations: + mariadb: + enabled: false + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + replicas: + server: 3 + prometheus_mysql_exporter: 1 + lifecycle: + upgrades: + deployments: + revision_history: 3 + pod_replacement_strategy: RollingUpdate + rolling_update: + max_unavailable: 1 + max_surge: 3 + termination_grace_period: + prometheus_mysql_exporter: + timeout: 30 + error_pages: + timeout: 10 + disruption_budget: + mariadb: + min_available: 0 + resources: + enabled: false + server: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + jobs: + tests: + limits: + memory: "1024Mi" + cpu: "2000m" + requests: + memory: "128Mi" + cpu: "100m" + image_repo_sync: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + mariadb_backup: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + ks_user: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - mariadb-server-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + mariadb_server_ks_user: + services: + - endpoint: internal + service: oslo_db + mariadb_backup: + services: + - endpoint: internal + service: oslo_db + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + tests: + services: + - endpoint: internal + service: oslo_db + +volume: + backup: + enabled: true + class_name: general + size: 5Gi + +jobs: + mariadb_backup: + # activeDeadlineSeconds == 0 means no deadline + activeDeadlineSeconds: 0 + backoffLimit: 6 + cron: "0 0 * * *" + history: + success: 3 + failed: 1 + ks_user: + # activeDeadlineSeconds == 0 means no deadline + activeDeadlineSeconds: 0 + backoffLimit: 6 + +conf: + mariadb_server: + setup_wait: + iteration: 30 + duration: 5 + database: + my: | + [mysqld] + datadir=/var/lib/mysql + basedir=/usr + ignore-db-dirs=lost+found + + [client-server] + !includedir /etc/mysql/conf.d/ + backup: + enabled: false + base_path: /var/backup + validateData: + ageOffset: 120 + mysqldump_options: > + --single-transaction --quick --add-drop-database + --add-drop-table --add-locks --databases + days_to_keep: 3 + remote_backup: + enabled: false + container_name: mariadb + days_to_keep: 14 + storage_policy: default-placement + number_of_retries: 5 + delay_range: + min: 30 + max: 60 + throttle_backups: + enabled: false + sessions_limit: 480 + lock_expire_after: 7200 + retry_after: 3600 + container_name: throttle-backups-manager + +secrets: + identity: + admin: keystone-admin-user + mariadb-server: mariadb-backup-user + mariadb: + backup_restore: mariadb-backup-restore + oci_image_registry: + mariadb: mariadb-oci-image-registry-key + tls: + oslo_db: + server: + public: mariadb-tls-server + internal: mariadb-tls-direct + +# typically overridden by environmental +# values, but should include all endpoints +# required by this chart +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + mariadb: + username: mariadb + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + oslo_db: + namespace: null + auth: + admin: + username: root + password: password + sst: + username: sst + password: password + audit: + username: audit + password: password + exporter: + username: exporter + password: password + hosts: + default: mariadb-server-primary + direct: mariadb-server-internal + discovery: mariadb-discovery + server: mariadb-server + host_fqdn_override: + default: null + path: null + scheme: mysql+pymysql + port: + mysql: + default: 3306 + wsrep: + default: 4567 + identity: + name: backup-storage-auth + namespace: openstack + auth: + admin: + # Auth URL of null indicates local authentication + # HTK will form the URL unless specified here + auth_url: null + region_name: RegionOne + username: admin + password: password + project_name: admin + user_domain_name: default + project_domain_name: default + mariadb: + # Auth URL of null indicates local authentication + # HTK will form the URL unless specified here + auth_url: null + role: admin + region_name: RegionOne + username: mariadb-backup-user + password: password + project_name: service + user_domain_name: service + project_domain_name: service + hosts: + default: keystone + internal: keystone-api + host_fqdn_override: + default: null + path: + default: /v3 + scheme: + default: 'http' + port: + api: + default: 80 + internal: 5000 + +network_policy: + mariadb: + ingress: + - {} + egress: + - {} + +# Helm hook breaks for helm2. +# Set helm3_hook: false in case helm2 is used. +helm3_hook: true + +manifests: + certificates: false + configmap_bin: true + configmap_etc: true + job_ks_user: false + cron_job_mariadb_backup: true + pvc_backup: true + network_policy: false + pod_test: true + secret_dbadmin_password: true + secret_sst_password: true + secret_dbaudit_password: true + secret_backup_restore: true + secret_etc: true + +... diff --git a/mariadb-cluster/.helmignore b/mariadb-cluster/.helmignore new file mode 100644 index 0000000000..f0c1319444 --- /dev/null +++ b/mariadb-cluster/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/mariadb-cluster/Chart.yaml b/mariadb-cluster/Chart.yaml new file mode 100644 index 0000000000..731dd1f8fa --- /dev/null +++ b/mariadb-cluster/Chart.yaml @@ -0,0 +1,31 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: v10.6.14 +description: OpenStack-Helm MariaDB controlled by mariadb-operator +name: mariadb-cluster +version: 2024.2.0 +home: https://mariadb.com/kb/en/ +icon: http://badges.mariadb.org/mariadb-badge-180x60.png +sources: + - https://github.com/MariaDB/server + - https://github.com/mariadb-operator/mariadb-operator + - https://opendev.org/openstack/openstack-helm +maintainers: + - name: OpenStack-Helm Authors +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit + version: ">= 0.1.0" +... diff --git a/mariadb-cluster/README.rst b/mariadb-cluster/README.rst new file mode 100644 index 0000000000..1615a9065c --- /dev/null +++ b/mariadb-cluster/README.rst @@ -0,0 +1,18 @@ +openstack-helm/mariadb +====================== + +By default, this chart creates a 3-member mariadb galera cluster. + +This chart depends on mariadb-operator chart. + +The StatefulSets all leverage PVCs to provide stateful storage to +``/var/lib/mysql``. + +You must ensure that your control nodes that should receive mariadb +instances are labeled with ``openstack-control-plane=enabled``, or +whatever you have configured in values.yaml for the label +configuration: + +:: + + kubectl label nodes openstack-control-plane=enabled --all diff --git a/mariadb-cluster/templates/bin/_liveness.sh.tpl b/mariadb-cluster/templates/bin/_liveness.sh.tpl new file mode 100644 index 0000000000..ca1df1d9c6 --- /dev/null +++ b/mariadb-cluster/templates/bin/_liveness.sh.tpl @@ -0,0 +1,68 @@ +#!/usr/bin/env bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -e + +MYSQL="mariadb \ + --defaults-file=/etc/mysql/admin_user.cnf \ + --host=localhost \ +{{- if .Values.manifests.certificates }} + --ssl-verify-server-cert=false \ + --ssl-ca=/etc/mysql/certs/ca.crt \ + --ssl-key=/etc/mysql/certs/tls.key \ + --ssl-cert=/etc/mysql/certs/tls.crt \ +{{- end }} + --connect-timeout 2" + +mysql_status_query () { + STATUS=$1 + $MYSQL -e "show status like \"${STATUS}\"" | \ + awk "/${STATUS}/ { print \$NF; exit }" +} + +{{- if eq (int .Values.pod.replicas.server) 1 }} +if ! $MYSQL -e 'select 1' > /dev/null 2>&1 ; then + exit 1 +fi + +{{- else }} +# if [ -f /var/lib/mysql/sst_in_progress ]; then +# # SST in progress, with this node receiving a snapshot. +# # MariaDB won't be up yet; avoid killing. +# exit 0 +# fi + +if [ "x$(mysql_status_query wsrep_ready)" != "xON" ]; then + # WSREP says the node can receive queries + exit 1 +fi + +if [ "x$(mysql_status_query wsrep_connected)" != "xON" ]; then + # WSREP connected + exit 1 +fi + +if [ "x$(mysql_status_query wsrep_cluster_status)" != "xPrimary" ]; then + # Not in primary cluster + exit 1 +fi + +wsrep_local_state_comment=$(mysql_status_query wsrep_local_state_comment) +if [ "x${wsrep_local_state_comment}" != "xSynced" ] && [ "x${wsrep_local_state_comment}" != "xDonor/Desynced" ]; then + # WSREP not synced or not sending SST + exit 1 +fi +{{- end }} diff --git a/mariadb-cluster/templates/bin/_readiness.sh.tpl b/mariadb-cluster/templates/bin/_readiness.sh.tpl new file mode 100644 index 0000000000..0ee233adbb --- /dev/null +++ b/mariadb-cluster/templates/bin/_readiness.sh.tpl @@ -0,0 +1,60 @@ +#!/usr/bin/env bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -e + +MYSQL="mariadb \ + --defaults-file=/etc/mysql/admin_user.cnf \ + --host=localhost \ +{{- if .Values.manifests.certificates }} + --ssl-verify-server-cert=false \ + --ssl-ca=/etc/mysql/certs/ca.crt \ + --ssl-key=/etc/mysql/certs/tls.key \ + --ssl-cert=/etc/mysql/certs/tls.crt \ +{{- end }} + --connect-timeout 2" + +mysql_status_query () { + STATUS=$1 + $MYSQL -e "show status like \"${STATUS}\"" | \ + awk "/${STATUS}/ { print \$NF; exit }" +} + +if ! $MYSQL -e 'select 1' > /dev/null 2>&1 ; then + exit 1 +fi + +{{- if gt (int .Values.pod.replicas.server) 1 }} +if [ "x$(mysql_status_query wsrep_ready)" != "xON" ]; then + # WSREP says the node can receive queries + exit 1 +fi + +if [ "x$(mysql_status_query wsrep_connected)" != "xON" ]; then + # WSREP connected + exit 1 +fi + +if [ "x$(mysql_status_query wsrep_cluster_status)" != "xPrimary" ]; then + # Not in primary cluster + exit 1 +fi + +if [ "x$(mysql_status_query wsrep_local_state_comment)" != "xSynced" ]; then + # WSREP not synced + exit 1 +fi +{{- end }} diff --git a/mariadb-cluster/templates/bin/_test.sh.tpl b/mariadb-cluster/templates/bin/_test.sh.tpl new file mode 100644 index 0000000000..536a4213e5 --- /dev/null +++ b/mariadb-cluster/templates/bin/_test.sh.tpl @@ -0,0 +1,27 @@ +#!/bin/bash +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +rm -f /tmp/test-success + +mysqlslap \ + --defaults-file=/etc/mysql/test-params.cnf \ + {{ include "helm-toolkit.utils.joinListWithSpace" $.Values.conf.tests.params }} -vv \ + --post-system="touch /tmp/test-success" + +if ! [ -f /tmp/test-success ]; then + exit 1 +fi diff --git a/mariadb-cluster/templates/certificates.yaml b/mariadb-cluster/templates/certificates.yaml new file mode 100644 index 0000000000..200f974acf --- /dev/null +++ b/mariadb-cluster/templates/certificates.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.certificates -}} +{{ dict "envAll" . "service" "oslo_db" "type" "default" | include "helm-toolkit.manifests.certificates" }} +{{- end -}} diff --git a/mariadb-cluster/templates/configmap-bin.yaml b/mariadb-cluster/templates/configmap-bin.yaml new file mode 100644 index 0000000000..6fac66a706 --- /dev/null +++ b/mariadb-cluster/templates/configmap-bin.yaml @@ -0,0 +1,41 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_bin }} +{{- $envAll := . }} +{{ if eq .Values.endpoints.oslo_db.auth.admin.username .Values.endpoints.oslo_db.auth.sst.username }} +{{ fail "the DB admin username should not match the sst user username" }} +{{ end }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: mariadb-bin +data: +{{- if .Values.images.local_registry.active }} + image-repo-sync.sh: | +{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} +{{- end }} +{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" ( index $envAll.Values.conf.database "init_script" ) "key" "init.sh" ) | indent 2 }} + readiness.sh: | +{{ tuple "bin/_readiness.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + liveness.sh: | +{{ tuple "bin/_liveness.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + test.sh: | +{{ tuple "bin/_test.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- if .Values.manifests.job_ks_user }} + ks-user.sh: | +{{ include "helm-toolkit.scripts.keystone_user" . | indent 4 }} +{{- end }} +{{- end }} diff --git a/mariadb-cluster/templates/configmap-etc.yaml b/mariadb-cluster/templates/configmap-etc.yaml new file mode 100644 index 0000000000..dc52daddc1 --- /dev/null +++ b/mariadb-cluster/templates/configmap-etc.yaml @@ -0,0 +1,24 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License" ); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_etc }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: mariadb-etc +data: +{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" ( index $envAll.Values.conf.database "my" ) "key" "my.cnf" ) | indent 2 }} +{{- end }} diff --git a/mariadb-cluster/templates/job-image-repo-sync.yaml b/mariadb-cluster/templates/job-image-repo-sync.yaml new file mode 100644 index 0000000000..2f59221ad7 --- /dev/null +++ b/mariadb-cluster/templates/job-image-repo-sync.yaml @@ -0,0 +1,22 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $serviceName := tuple "oslo_db" "server" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" $serviceName -}} +{{- if .Values.pod.tolerations.mariadb.enabled -}} +{{- $_ := set $imageRepoSyncJob "tolerationsEnabled" true -}} +{{- end -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/mariadb-cluster/templates/job-refresh-statefulset.yaml b/mariadb-cluster/templates/job-refresh-statefulset.yaml new file mode 100644 index 0000000000..b16a73035e --- /dev/null +++ b/mariadb-cluster/templates/job-refresh-statefulset.yaml @@ -0,0 +1,105 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.mariadb }} +{{- $envAll := . }} + +{{- $serviceAccountName := "mariadb-cluster-refresh-statefulset" }} +{{ tuple $envAll "mariadb_cluster_refresh_statefulset" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +rules: + - apiGroups: + - "" + - extensions + - batch + - apps + resources: + - statefulsets + verbs: + - get + - list + - delete +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: mariadb-cluster-refresh-statefulset + labels: +{{ tuple $envAll "mariadb-cluster" "refresh-statefulset" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- if .Values.helm3_hook }} + annotations: + "helm.sh/hook": "post-upgrade" + "helm.sh/hook-weight": "5" + "helm.sh/hook-delete-policy": "before-hook-creation" +{{- end }} +spec: + backoffLimit: {{ .Values.jobs.mariadb_cluster_refresh_statefulset.backoffLimit }} + template: + metadata: + labels: +{{ tuple $envAll "mariadb-cluster" "refresh-statefulset" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} +{{ dict "envAll" $envAll "podName" "" "containerNames" (list "init" "exporter-create-sql-user") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: + shareProcessNamespace: true + serviceAccountName: {{ $serviceAccountName }} +{{ dict "envAll" $envAll "application" "job" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + activeDeadlineSeconds: {{ .Values.jobs.mariadb_cluster_refresh_statefulset.activeDeadlineSeconds }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} + initContainers: +{{ tuple $envAll "mariadb_cluster_refresh_statefulset" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: refresh-statefulset +{{ tuple $envAll "mariadb_cluster_refresh_statefulset" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ dict "envAll" $envAll "application" "mariadb_cluster_refresh_statefulset" "container" "main" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.mariadb_cluster_refresh_statefulset | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + command: ["/bin/sh", "-c"] + args: ["kubectl delete statefulset ${STATEFULSET_NAME} --namespace=${NAMESPACE}"] + env: + - name: STATEFULSET_NAME + value: {{ tuple "oslo_db" "server" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + - name: NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + volumeMounts: + - name: pod-tmp + mountPath: /tmp + volumes: + - name: pod-tmp + emptyDir: {} +{{- end }} diff --git a/mariadb-cluster/templates/mariadb.yaml b/mariadb-cluster/templates/mariadb.yaml new file mode 100644 index 0000000000..56bd348dad --- /dev/null +++ b/mariadb-cluster/templates/mariadb.yaml @@ -0,0 +1,215 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "mariadbReadinessProbe" }} +exec: + command: + - /tmp/readiness.sh +{{- end }} +{{- define "mariadbLivenessProbe" }} +exec: + command: + - /tmp/liveness.sh +{{- end }} + +{{- if (.Values.global).subchart_release_name }} +{{- $_ := set . "deployment_name" .Chart.Name }} +{{- else }} +{{- $_ := set . "deployment_name" .Release.Name }} +{{- end }} + +{{- if .Values.manifests.mariadb }} +{{- $envAll := . }} + +--- +apiVersion: mariadb.mmontes.io/v1alpha1 +kind: MariaDB +metadata: + # NOTE(portdirect): the statefulset name must match the POD_NAME_PREFIX env var for discovery to work + name: {{ tuple "oslo_db" "server" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + mariadb-dbadmin-password-hash: {{ tuple "secret-dbadmin-password.yaml" . | include "helm-toolkit.utils.hash" }} + labels: +{{ tuple $envAll "mariadb" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + rootPasswordSecretKeyRef: + name: mariadb-dbadmin-password + key: MYSQL_DBADMIN_PASSWORD + +{{ tuple $envAll "mariadb" | include "helm-toolkit.snippets.image" | indent 2 }} + + initContainers: + - command: + - /tmp/init.sh +{{ tuple $envAll "mariadb" | include "helm-toolkit.snippets.image" | indent 6 }} +{{ dict "envAll" $envAll "application" "server" "container" "perms" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 6 }} +{{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 6 }} + +{{ if $envAll.Values.conf.galera.enabled }} + galera: + enabled: true + primary: + podIndex: {{ .Values.conf.galera.primary.podIndex }} + automaticFailover: {{ .Values.conf.galera.primary.automaticFailover }} + sst: {{ .Values.conf.galera.sst }} + replicaThreads: {{ .Values.conf.galera.replicaThreads }} + agent: +{{ tuple $envAll "agent" | include "helm-toolkit.snippets.image" | indent 6 }} +{{- dict "envAll" $envAll "application" "server" "container" "agent" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 6 }} + args: + - '--graceful-shutdown-timeout=5s' + - '--recovery-timeout=5m0s' + - '-log-dev' + - '-log-level=debug' + port: {{ .Values.conf.galera.agent.port }} + {{- if $envAll.Values.conf.galera.agent.kubernetesAuth.enabled }} + kubernetesAuth: + enabled: true + {{- end }} + gracefulShutdownTimeout: {{ .Values.conf.galera.agent.gracefulShutdownTimeout }} + {{- if $envAll.Values.conf.galera.recovery.enabled }} + recovery: + enabled: true + clusterHealthyTimeout: {{ .Values.conf.galera.recovery.clusterHealthyTimeout }} + clusterBootstrapTimeout: {{ .Values.conf.galera.recovery.clusterBootstrapTimeout }} + podRecoveryTimeout: {{ .Values.conf.galera.recovery.podRecoveryTimeout }} + podSyncTimeout: {{ .Values.conf.galera.recovery.podSyncTimeout }} + {{- end }} + initContainer: +{{ tuple $envAll "initContainer" | include "helm-toolkit.snippets.image" | indent 6 }} +{{- dict "envAll" $envAll "application" "server" "container" "init" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 6 }} + args: + - '-log-dev' + - '-log-level=debug' + # galera volume templates + volumeClaimTemplate: + resources: + requests: + storage: {{ .Values.volume.galera.size }} + accessModes: + - ReadWriteOnce + storageClassName: {{ .Values.volume.galera.class_name }} +{{ end }} + +{{ include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" ( index $envAll.Values.conf.database "galera" ) "key" "myCnf" ) | indent 2 }} + + replicas: {{ .Values.pod.replicas.server }} + + affinity: +{{- tuple $envAll "mariadb" "server" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 4 }} + +{{ if $envAll.Values.pod.tolerations.mariadb.enabled }} +{{- tuple $envAll "mariadb" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 2 }} +{{- end }} + + updateStrategy: + type: {{ .Values.pod.lifecycle.upgrades.deployments.pod_replacement_strategy }} + +{{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 2 }} +{{ dict "envAll" $envAll "application" "server" "container" "mariadb" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 2 }} + + nodeSelector: + {{ .Values.labels.server.node_selector_key }}: {{ .Values.labels.server.node_selector_value }} + + podAnnotations: +{{- dict "envAll" $envAll "podName" "mariadb-server" "containerNames" (list "init-0" "init" "agent" "mariadb") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 4 }} + + podDisruptionBudget: + minAvailable: {{ .Values.pod.lifecycle.disruption_budget.mariadb.min_available }} + +{{ dict "envAll" . "component" "server" "container" "mariadb" "type" "readiness" "probeTemplate" (include "mariadbReadinessProbe" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 2 }} + +{{ dict "envAll" . "component" "server" "container" "mariadb" "type" "liveness" "probeTemplate" (include "mariadbLivenessProbe" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 2 }} + +{{ if .Values.monitoring.prometheus.enabled }} + metrics: + exporter: +{{ tuple $envAll "prometheus_mysql_exporter" | include "helm-toolkit.snippets.image" | indent 6 }} +{{ dict "envAll" $envAll "application" "prometheus_mysql_exporter" "container" "exporter" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 6 }} +{{ tuple $envAll $envAll.Values.pod.resources.prometheus_mysql_exporter | include "helm-toolkit.snippets.kubernetes_resources" | indent 6 }} + port: {{ tuple "prometheus_mysql_exporter" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +{{- if $envAll.Values.manifests.certificates }} + volumeMounts: +{{ dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.secrets.tls.oslo_db.server.internal "path" "/etc/mysql/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 8 }} +{{- end }} + serviceMonitor: + prometheusRelease: prometheus-mysql-exporter + interval: 10s + scrapeTimeout: 10s +{{ end }} + + env: + - name: MYSQL_HISTFILE + value: {{ .Values.conf.database.mysql_histfile }} +{{ if .Values.conf.database.auto_upgrade.enabled }} + - name: MARIADB_AUTO_UPGRADE + value: {{ .Values.conf.database.auto_upgrade.enabled | quote }} + - name: MARIADB_DISABLE_UPGRADE_BACKUP + value: {{ .Values.conf.database.auto_upgrade.disable_upgrade_backup | quote }} +{{ end }} + + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: mariadb-secrets + mountPath: /etc/mysql/admin_user.cnf + subPath: admin_user.cnf + readOnly: true + - name: mariadb-secrets + mountPath: /docker-entrypoint-initdb.d/privileges.sql + subPath: privileges.sql + readOnly: true + - name: mariadb-bin + mountPath: /tmp/init.sh + subPath: init.sh + - name: mariadb-bin + mountPath: /tmp/readiness.sh + subPath: readiness.sh + readOnly: true + - name: mariadb-bin + mountPath: /tmp/liveness.sh + subPath: liveness.sh + readOnly: true +{{ dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.secrets.tls.oslo_db.server.internal "path" "/etc/mysql/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 4 }} + + volumes: + - name: pod-tmp + emptyDir: {} + - name: mariadb-bin + configMap: + name: mariadb-bin + defaultMode: 0555 + - name: mariadb-etc + configMap: + name: mariadb-etc + defaultMode: 0444 + - name: mariadb-secrets + secret: + secretName: mariadb-secrets + defaultMode: 0444 +{{ dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.secrets.tls.oslo_db.server.internal | include "helm-toolkit.snippets.tls_volume" | indent 4 }} + + # storage volume templates + volumeClaimTemplate: + resources: + requests: + storage: {{ .Values.volume.size }} + accessModes: + - ReadWriteOnce + {{- if ne .Values.volume.class_name "default" }} + storageClassName: {{ .Values.volume.class_name }} + {{- end }} + +{{- end }} diff --git a/mariadb-cluster/templates/network_policy.yaml b/mariadb-cluster/templates/network_policy.yaml new file mode 100644 index 0000000000..78ecc07bd0 --- /dev/null +++ b/mariadb-cluster/templates/network_policy.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{- if .Values.manifests.network_policy -}} +{{- $netpol_opts := dict "envAll" . "name" "application" "label" "mariadb" -}} +{{ $netpol_opts | include "helm-toolkit.manifests.kubernetes_network_policy" }} +{{- end -}} diff --git a/mariadb-cluster/templates/pod-test.yaml b/mariadb-cluster/templates/pod-test.yaml new file mode 100644 index 0000000000..c8b3c29c37 --- /dev/null +++ b/mariadb-cluster/templates/pod-test.yaml @@ -0,0 +1,86 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if (.Values.global).subchart_release_name }} +{{- $_ := set . "deployment_name" .Chart.Name }} +{{- else }} +{{- $_ := set . "deployment_name" .Release.Name }} +{{- end }} + +{{- if .Values.manifests.pod_test }} +{{- $envAll := . }} +{{- $dependencies := .Values.dependencies.static.tests }} + +{{- $serviceAccountName := print .deployment_name "-test" }} +{{ tuple $envAll "tests" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: v1 +kind: Pod +metadata: + name: "{{.deployment_name}}-test" + labels: +{{ tuple $envAll "mariadb" "test" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: + "helm.sh/hook": test-success + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +{{ dict "envAll" $envAll "podName" "mariadb-test" "containerNames" (list "init" "mariadb-test") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 4 }} +spec: + shareProcessNamespace: true + serviceAccountName: {{ $serviceAccountName }} +{{ dict "envAll" $envAll "application" "tests" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 2 }} +{{ if $envAll.Values.pod.tolerations.mariadb.enabled }} +{{ tuple $envAll "mariadb" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 2 }} +{{ end }} + nodeSelector: + {{ .Values.labels.test.node_selector_key }}: {{ .Values.labels.test.node_selector_value }} + restartPolicy: Never + initContainers: +{{ tuple $envAll "tests" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 4 }} + containers: + - name: mariadb-test +{{ dict "envAll" $envAll "application" "tests" "container" "test" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 6 }} +{{ tuple $envAll "scripted_test" | include "helm-toolkit.snippets.image" | indent 6 }} + command: + - /tmp/test.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: mariadb-bin + mountPath: /tmp/test.sh + subPath: test.sh + readOnly: true + - name: mariadb-secrets + mountPath: /etc/mysql/test-params.cnf + {{ if eq $envAll.Values.conf.tests.endpoint "internal" }} + subPath: admin_user_internal.cnf + {{ else if eq $envAll.Values.conf.tests.endpoint "direct" }} + subPath: admin_user.cnf + {{ else }} + {{ fail "Either 'direct' or 'internal' should be specified for .Values.conf.tests.endpoint" }} + {{ end }} + readOnly: true +{{ dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.secrets.tls.oslo_db.server.internal "path" "/etc/mysql/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 8 }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: mariadb-bin + configMap: + name: mariadb-bin + defaultMode: 0555 + - name: mariadb-secrets + secret: + secretName: mariadb-secrets + defaultMode: 0444 +{{ dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.secrets.tls.oslo_db.server.internal | include "helm-toolkit.snippets.tls_volume" | indent 4 }} +{{- end }} diff --git a/mariadb-cluster/templates/secret-dbadmin-password.yaml b/mariadb-cluster/templates/secret-dbadmin-password.yaml new file mode 100644 index 0000000000..c9f8c4e268 --- /dev/null +++ b/mariadb-cluster/templates/secret-dbadmin-password.yaml @@ -0,0 +1,25 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_dbadmin_password }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: mariadb-dbadmin-password +type: Opaque +data: + MYSQL_DBADMIN_PASSWORD: {{ .Values.endpoints.oslo_db.auth.admin.password | b64enc }} +{{- end }} diff --git a/mariadb-cluster/templates/secret-dbaudit-password.yaml b/mariadb-cluster/templates/secret-dbaudit-password.yaml new file mode 100644 index 0000000000..7733da7dd3 --- /dev/null +++ b/mariadb-cluster/templates/secret-dbaudit-password.yaml @@ -0,0 +1,25 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_dbaudit_password }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: mariadb-dbaudit-password +type: Opaque +data: + MYSQL_DBAUDIT_PASSWORD: {{ .Values.endpoints.oslo_db.auth.audit.password | b64enc }} +{{- end }} diff --git a/mariadb-cluster/templates/secret-registry.yaml b/mariadb-cluster/templates/secret-registry.yaml new file mode 100644 index 0000000000..da979b3223 --- /dev/null +++ b/mariadb-cluster/templates/secret-registry.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_registry .Values.endpoints.oci_image_registry.auth.enabled }} +{{ include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) }} +{{- end }} diff --git a/mariadb-cluster/templates/secret-sst-password.yaml b/mariadb-cluster/templates/secret-sst-password.yaml new file mode 100644 index 0000000000..c49c0ff9b8 --- /dev/null +++ b/mariadb-cluster/templates/secret-sst-password.yaml @@ -0,0 +1,25 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_sst_password }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: mariadb-dbsst-password +type: Opaque +data: + MYSQL_DBSST_PASSWORD: {{ .Values.endpoints.oslo_db.auth.sst.password | b64enc }} +{{- end }} diff --git a/mariadb-cluster/templates/secrets-etc.yaml b/mariadb-cluster/templates/secrets-etc.yaml new file mode 100644 index 0000000000..51bafd3223 --- /dev/null +++ b/mariadb-cluster/templates/secrets-etc.yaml @@ -0,0 +1,27 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_etc }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: mariadb-secrets +type: Opaque +data: + admin_user.cnf: {{ tuple "secrets/_admin_user.cnf.tpl" . | include "helm-toolkit.utils.template" | b64enc }} + admin_user_internal.cnf: {{ tuple "secrets/_admin_user_internal.cnf.tpl" . | include "helm-toolkit.utils.template" | b64enc }} + privileges.sql: {{ tuple "secrets/_privileges.sql.tpl" . | include "helm-toolkit.utils.template" | b64enc }} +{{- end }} diff --git a/mariadb-cluster/templates/secrets/_admin_user.cnf.tpl b/mariadb-cluster/templates/secrets/_admin_user.cnf.tpl new file mode 100644 index 0000000000..0031a4bd7d --- /dev/null +++ b/mariadb-cluster/templates/secrets/_admin_user.cnf.tpl @@ -0,0 +1,24 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +[client] +user = {{ .Values.endpoints.oslo_db.auth.admin.username }} +password = {{ .Values.endpoints.oslo_db.auth.admin.password }} +host = {{ tuple "oslo_db" "direct" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +port = {{ tuple "oslo_db" "direct" "mysql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +{{- if .Values.manifests.certificates }} +ssl-ca = /etc/mysql/certs/ca.crt +ssl-key = /etc/mysql/certs/tls.key +ssl-cert = /etc/mysql/certs/tls.crt +{{- end }} diff --git a/mariadb-cluster/templates/secrets/_admin_user_internal.cnf.tpl b/mariadb-cluster/templates/secrets/_admin_user_internal.cnf.tpl new file mode 100644 index 0000000000..fa0d09a559 --- /dev/null +++ b/mariadb-cluster/templates/secrets/_admin_user_internal.cnf.tpl @@ -0,0 +1,24 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +[client] +user = {{ .Values.endpoints.oslo_db.auth.admin.username }} +password = {{ .Values.endpoints.oslo_db.auth.admin.password }} +host = {{ tuple "oslo_db" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }} +port = {{ tuple "oslo_db" "internal" "mysql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +{{- if .Values.manifests.certificates }} +ssl-ca = /etc/mysql/certs/ca.crt +ssl-key = /etc/mysql/certs/tls.key +ssl-cert = /etc/mysql/certs/tls.crt +{{- end }} diff --git a/mariadb-cluster/templates/secrets/_privileges.sql.tpl b/mariadb-cluster/templates/secrets/_privileges.sql.tpl new file mode 100644 index 0000000000..01d3f9a66d --- /dev/null +++ b/mariadb-cluster/templates/secrets/_privileges.sql.tpl @@ -0,0 +1,20 @@ +########################################### +# The lines not confirmed to be working with operator are disabled +########################################### +# DELETE FROM mysql.user WHERE user != 'mariadb.sys'; +# CREATE OR REPLACE USER '{{ .Values.endpoints.oslo_db.auth.admin.username }}'@'%' IDENTIFIED BY '{{ .Values.endpoints.oslo_db.auth.admin.password }}'; +{{- if .Values.manifests.certificates }} +GRANT ALL ON *.* TO '{{ .Values.endpoints.oslo_db.auth.admin.username }}'@'%' REQUIRE X509 WITH GRANT OPTION; +{{- else }} +GRANT ALL ON *.* TO '{{ .Values.endpoints.oslo_db.auth.admin.username }}'@'%' WITH GRANT OPTION; +{{- end }} +DROP DATABASE IF EXISTS test ; +# CREATE OR REPLACE USER '{{ .Values.endpoints.oslo_db.auth.sst.username }}'@'127.0.0.1' IDENTIFIED BY '{{ .Values.endpoints.oslo_db.auth.sst.password }}'; +# GRANT PROCESS, RELOAD, LOCK TABLES, REPLICATION CLIENT ON *.* TO '{{ .Values.endpoints.oslo_db.auth.sst.username }}'@'127.0.0.1' ; +CREATE OR REPLACE USER '{{ .Values.endpoints.oslo_db.auth.audit.username }}'@'%' IDENTIFIED BY '{{ .Values.endpoints.oslo_db.auth.audit.password }}'; +{{- if .Values.manifests.certificates }} +GRANT SELECT ON *.* TO '{{ .Values.endpoints.oslo_db.auth.audit.username }}'@'%' REQUIRE X509; +{{- else }} +GRANT SELECT ON *.* TO '{{ .Values.endpoints.oslo_db.auth.audit.username }}'@'%' ; +{{- end }} +FLUSH PRIVILEGES ; diff --git a/mariadb-cluster/values.yaml b/mariadb-cluster/values.yaml new file mode 100644 index 0000000000..c3076a9f3b --- /dev/null +++ b/mariadb-cluster/values.yaml @@ -0,0 +1,581 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for mariadb. +# This is a YAML-formatted file. +# Declare name/value pairs to be passed into your templates. +# name: value + +--- +release_group: null + +images: + tags: + agent: ghcr.io/mariadb-operator/agent:v0.0.3 + initContainer: ghcr.io/mariadb-operator/init:v0.0.6 + mariadb: docker.io/library/mariadb:10.6.14-focal + prometheus_mysql_exporter: docker.io/prom/mysqld-exporter:v0.12.1 + prometheus_mysql_exporter_helm_tests: docker.io/openstackhelm/heat:wallaby-ubuntu_focal + dep_check: quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal + image_repo_sync: docker.io/library/docker:17.07.0 + scripted_test: docker.io/library/mariadb:10.6.14-focal + mariadb_cluster_refresh_statefulset: quay.io/airshipit/porthole-mysqlclient-utility:latest-ubuntu_focal + pull_policy: "IfNotPresent" + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +labels: + server: + node_selector_key: openstack-control-plane + node_selector_value: enabled + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + test: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +pod: + probes: + server: + mariadb: + readiness: + enabled: true + params: + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 15 + liveness: + enabled: true + params: + initialDelaySeconds: 120 + periodSeconds: 30 + timeoutSeconds: 15 + security_context: + server: + pod: + runAsUser: 0 + container: + init-0: + runAsUser: 0 + readOnlyRootFilesystem: true + init: + runAsUser: 0 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false + agent: + runAsUser: 0 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false + mariadb: + runAsUser: 0 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: false + mariadb_cluster_refresh_statefulset: + pod: + runAsUser: 0 + container: + main: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + tests: + pod: + runAsUser: 999 + container: + test: + runAsUser: 999 + readOnlyRootFilesystem: true + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + weight: + default: 10 + tolerations: + mariadb: + enabled: false + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + replicas: + server: 3 + prometheus_mysql_exporter: 1 + lifecycle: + upgrades: + deployments: + revision_history: 3 + pod_replacement_strategy: RollingUpdate + rolling_update: + max_unavailable: 1 + max_surge: 3 + termination_grace_period: + prometheus_mysql_exporter: + timeout: 30 + disruption_budget: + mariadb: + min_available: 0 + resources: + enabled: false + server: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + jobs: + tests: + limits: + memory: "1024Mi" + cpu: "2000m" + requests: + memory: "128Mi" + cpu: "100m" + mariadb_cluster_refresh_statefulset: + limits: + memory: "1024Mi" + cpu: "2000m" + requests: + memory: "128Mi" + cpu: "100m" + image_repo_sync: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - mariadb-server-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + tests: + services: + - endpoint: internal + service: oslo_db + +volume: + enabled: true + class_name: general + size: 5Gi + backup: + enabled: true + class_name: general + size: 5Gi + galera: + enabled: true + class_name: general + size: 300Mi + +jobs: + mariadb_cluster_refresh_statefulset: + backoffLimit: 87600 + activeDeadlineSeconds: 3600 + +conf: + galera: + enabled: true + primary: + podIndex: 0 + automaticFailover: true + sst: mariabackup + replicaThreads: 1 + agent: + port: 5555 + kubernetesAuth: + enabled: true + gracefulShutdownTimeout: 5s + recovery: + enabled: true + clusterHealthyTimeout: 3m + clusterBootstrapTimeout: 10m + podRecoveryTimeout: 5m + podSyncTimeout: 5m + tests: + # This may either be: + # * internal: which will hit the endpoint exposed by the ingress controller + # * direct: which will hit the backends directly via a k8s service ip + # Note, deadlocks and failure are to be expected with concurrency if + # hitting the `direct` endpoint. + endpoint: internal + # This is a list of tuning params passed to mysqlslap: + params: + - --auto-generate-sql + - --concurrency=100 + - --number-of-queries=1000 + - --number-char-cols=1 + - --number-int-cols=1 + mariadb_server: + setup_wait: + iteration: 30 + duration: 5 + database: + auto_upgrade: + enabled: true + disable_upgrade_backup: false + mysql_histfile: "/dev/null" + init_script: | + #!/usr/bin/env bash + + {{/* + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + */}} + + set -x + + chown -R "mysql:mysql" /var/lib/mysql; + chmod 771 /var/lib/mysql; + galera: | + [mariadb] + bind-address=0.0.0.0 + default_storage_engine=InnoDB + binlog_format=row + innodb_autoinc_lock_mode=2 + max_allowed_packet=256M + ######################## + # + ######################## + ignore-db-dirs=lost+found + + # Charset + character_set_server=utf8 + collation_server=utf8_general_ci + skip-character-set-client-handshake + + # Logging + slow_query_log=off + slow_query_log_file=/var/log/mysql/mariadb-slow.log + log_warnings=2 + + # General logging has huge performance penalty therefore is disabled by default + general_log=off + general_log_file=/var/log/mysql/mariadb-error.log + + long_query_time=3 + log_queries_not_using_indexes=on + + # Networking + bind_address=0.0.0.0 + port={{ tuple "oslo_db" "direct" "mysql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + + # When a client connects, the server will perform hostname resolution, + # and when DNS is slow, establishing the connection will become slow as well. + # It is therefore recommended to start the server with skip-name-resolve to + # disable all DNS lookups. The only limitation is that the GRANT statements + # must then use IP addresses only. + skip_name_resolve + + # Tuning + user=mysql + max_allowed_packet=256M + open_files_limit=10240 + max_connections=8192 + max-connect-errors=1000000 + + # General security settings + # Reference: https://dev.mysql.com/doc/mysql-security-excerpt/8.0/en/general-security-issues.html + # secure_file_priv is set to '/home' because it is read-only, which will + # disable this feature completely. + secure_file_priv=/home + local_infile=0 + symbolic_links=0 + sql_mode="STRICT_ALL_TABLES,STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION" + + + ## Generally, it is unwise to set the query cache to be larger than 64-128M + ## as the costs associated with maintaining the cache outweigh the performance + ## gains. + ## The query cache is a well known bottleneck that can be seen even when + ## concurrency is moderate. The best option is to disable it from day 1 + ## by setting query_cache_size=0 (now the default on MySQL 5.6) + ## and to use other ways to speed up read queries: good indexing, adding + ## replicas to spread the read load or using an external cache. + query_cache_size=0 + query_cache_type=0 + + sync_binlog=0 + thread_cache_size=16 + table_open_cache=2048 + table_definition_cache=1024 + + # + # InnoDB + # + # The buffer pool is where data and indexes are cached: having it as large as possible + # will ensure you use memory and not disks for most read operations. + # Typical values are 50..75% of available RAM. + # TODO(tomasz.paszkowski): This needs to by dynamic based on available RAM. + innodb_buffer_pool_size=1024M + innodb_doublewrite=0 + innodb_file_per_table=1 + innodb_flush_method=O_DIRECT + innodb_io_capacity=500 + innodb_log_file_size=128M + innodb_old_blocks_time=1000 + innodb_read_io_threads=8 + innodb_write_io_threads=8 + + {{ if .Values.manifests.certificates }} + # TLS + ssl_ca=/etc/mysql/certs/ca.crt + ssl_key=/etc/mysql/certs/tls.key + ssl_cert=/etc/mysql/certs/tls.crt + # tls_version = TLSv1.2,TLSv1.3 + {{ end }} + + + [mysqldump] + max-allowed-packet=16M + + [client] + default_character_set=utf8 + {{ if .Values.manifests.certificates }} + # TLS + ssl_ca=/etc/mysql/certs/ca.crt + ssl_key=/etc/mysql/certs/tls.key + ssl_cert=/etc/mysql/certs/tls.crt + # tls_version = TLSv1.2,TLSv1.3 + {{ end }} + + my: | + [mysqld] + datadir=/var/lib/mysql + basedir=/usr + ignore-db-dirs=lost+found + + [client-server] + !includedir /etc/mysql/conf.d/ + + config_override: null + # Any configuration here will override the base config. + # config_override: |- + # [mysqld] + # wsrep_slave_threads=1 + +monitoring: + prometheus: + enabled: false + mysqld_exporter: + scrape: true + +secrets: + identity: + admin: keystone-admin-user + oci_image_registry: + mariadb: mariadb-oci-image-registry-key + tls: + oslo_db: + server: + public: mariadb-tls-server + internal: mariadb-tls-direct + +# typically overridden by environmental +# values, but should include all endpoints +# required by this chart +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + mariadb: + username: mariadb + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + monitoring: + name: prometheus + namespace: null + hosts: + default: prom-metrics + public: prometheus + host_fqdn_override: + default: null + path: + default: null + scheme: + default: 'http' + port: + api: + default: 9090 + public: 80 + prometheus_mysql_exporter: + namespace: null + hosts: + default: mysql-exporter + host_fqdn_override: + default: null + path: + default: /metrics + scheme: + default: 'http' + port: + metrics: + default: 9104 + oslo_db: + namespace: null + auth: + admin: + username: root + password: password + sst: + username: sst + password: password + audit: + username: audit + password: password + exporter: + username: exporter + password: password + hosts: + default: mariadb-server-primary + direct: mariadb-server-internal + discovery: mariadb-discovery + server: mariadb-server + host_fqdn_override: + default: null + path: null + scheme: mysql+pymysql + port: + mysql: + default: 3306 + wsrep: + default: 4567 + kube_dns: + namespace: kube-system + name: kubernetes-dns + hosts: + default: kube-dns + host_fqdn_override: + default: null + path: + default: null + scheme: http + port: + dns_tcp: + default: 53 + dns: + default: 53 + protocol: UDP + identity: + name: backup-storage-auth + namespace: openstack + auth: + admin: + # Auth URL of null indicates local authentication + # HTK will form the URL unless specified here + auth_url: null + region_name: RegionOne + username: admin + password: password + project_name: admin + user_domain_name: default + project_domain_name: default + mariadb-server: + # Auth URL of null indicates local authentication + # HTK will form the URL unless specified here + auth_url: null + role: admin + region_name: RegionOne + username: mariadb-backup-user + password: password + project_name: service + user_domain_name: service + project_domain_name: service + hosts: + default: keystone + internal: keystone-api + host_fqdn_override: + default: null + path: + default: /v3 + scheme: + default: 'http' + port: + api: + default: 80 + internal: 5000 + +network_policy: + mariadb: + ingress: + - {} + egress: + - {} + +# Helm hook breaks for helm2. +# Set helm3_hook: false in case helm2 is used. +helm3_hook: true + +manifests: + certificates: false + configmap_bin: true + configmap_etc: true + job_image_repo_sync: true + network_policy: false + pod_test: true + secret_dbadmin_password: true + secret_sst_password: true + secret_dbaudit_password: true + secret_etc: true + secret_registry: true + service_primary: true + mariadb: true +... diff --git a/mariadb/.helmignore b/mariadb/.helmignore new file mode 100644 index 0000000000..f0c1319444 --- /dev/null +++ b/mariadb/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/mariadb/Chart.yaml b/mariadb/Chart.yaml new file mode 100644 index 0000000000..d8898d9460 --- /dev/null +++ b/mariadb/Chart.yaml @@ -0,0 +1,30 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: v10.6.7 +description: OpenStack-Helm MariaDB +name: mariadb +version: 2024.2.0 +home: https://mariadb.com/kb/en/ +icon: http://badges.mariadb.org/mariadb-badge-180x60.png +sources: + - https://github.com/MariaDB/server + - https://opendev.org/openstack/openstack-helm +maintainers: + - name: OpenStack-Helm Authors +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit + version: ">= 0.1.0" +... diff --git a/mariadb/README.rst b/mariadb/README.rst new file mode 100644 index 0000000000..93af0868a1 --- /dev/null +++ b/mariadb/README.rst @@ -0,0 +1,38 @@ +openstack-helm/mariadb +====================== + +By default, this chart creates a 3-member mariadb galera cluster. + +This chart leverages StatefulSets, with persistent storage. + +It creates a job that acts as a temporary standalone galera cluster. +This host is bootstrapped with authentication and then the WSREP +bindings are exposed publicly. The cluster members being StatefulSets +are provisioned one at a time. The first host must be marked as +``Ready`` before the next host will be provisioned. This is determined +by the readinessProbes which actually validate that MySQL is up and +responsive. + +The configuration leverages xtrabackup-v2 for synchronization. This may +later be augmented to leverage rsync which has some benefits. + +Once the seed job completes, which completes only when galera reports +that it is Synced and all cluster members are reporting in thus matching +the cluster count according to the job to the replica count in the helm +values configuration, the job is terminated. When the job is no longer +active, future StatefulSets provisioned will leverage the existing +cluster members as gcomm endpoints. It is only when the job is running +that the cluster members leverage the seed job as their gcomm endpoint. +This ensures you can restart members and scale the cluster. + +The StatefulSets all leverage PVCs to provide stateful storage to +``/var/lib/mysql``. + +You must ensure that your control nodes that should receive mariadb +instances are labeled with ``openstack-control-plane=enabled``, or +whatever you have configured in values.yaml for the label +configuration: + +:: + + kubectl label nodes openstack-control-plane=enabled --all diff --git a/mariadb/templates/bin/_backup_mariadb.sh.tpl b/mariadb/templates/bin/_backup_mariadb.sh.tpl new file mode 100644 index 0000000000..44db641420 --- /dev/null +++ b/mariadb/templates/bin/_backup_mariadb.sh.tpl @@ -0,0 +1,584 @@ +#!/bin/bash + +SCOPE=${1:-"all"} + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +source /tmp/backup_main.sh + +# Export the variables required by the framework +# Note: REMOTE_BACKUP_ENABLED, STORAGE_POLICY and CONTAINER_NAME are already +# exported. +export DB_NAMESPACE=${MARIADB_POD_NAMESPACE} +export DB_NAME="mariadb" +export LOCAL_DAYS_TO_KEEP=${MARIADB_LOCAL_BACKUP_DAYS_TO_KEEP} +export REMOTE_DAYS_TO_KEEP=${MARIADB_REMOTE_BACKUP_DAYS_TO_KEEP} +export REMOTE_BACKUP_RETRIES=${NUMBER_OF_RETRIES_SEND_BACKUP_TO_REMOTE} +export MIN_DELAY_SEND_REMOTE=${MIN_DELAY_SEND_BACKUP_TO_REMOTE} +export MAX_DELAY_SEND_REMOTE=${MAX_DELAY_SEND_BACKUP_TO_REMOTE} +export ARCHIVE_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${DB_NAMESPACE}/${DB_NAME}/archive + +# Dump all the database files to existing $TMP_DIR and save logs to $LOG_FILE +dump_databases_to_directory() { + TMP_DIR=$1 + LOG_FILE=$2 + SCOPE=${3:-"all"} + + + MYSQL="mysql \ + --defaults-file=/etc/mysql/admin_user.cnf \ + --connect-timeout 10" + + MYSQLDUMP="mysqldump \ + --defaults-file=/etc/mysql/admin_user.cnf" + + if [[ "${SCOPE}" == "all" ]]; then + MYSQL_DBNAMES=( $($MYSQL --silent --skip-column-names -e \ + "show databases;" | \ + grep -ivE 'information_schema|performance_schema|mysql|sys') ) + else + if [[ "${SCOPE}" != "information_schema" && "${SCOPE}" != "performance_schema" && "${SCOPE}" != "mysql" && "${SCOPE}" != "sys" ]]; then + MYSQL_DBNAMES=( ${SCOPE} ) + else + log ERROR "It is not allowed to backup database ${SCOPE}." + return 1 + fi + fi + + #check if there is a database to backup, otherwise exit + if [[ -z "${MYSQL_DBNAMES// }" ]] + then + log INFO "There is no database to backup" + return 0 + fi + + #Create a list of Databases + printf "%s\n" "${MYSQL_DBNAMES[@]}" > $TMP_DIR/db.list + + if [[ "${SCOPE}" == "all" ]]; then + #Retrieve and create the GRANT file for all the users +{{- if .Values.manifests.certificates }} + SSL_DSN=";mysql_ssl=1" + SSL_DSN="$SSL_DSN;mysql_ssl_client_key=/etc/mysql/certs/tls.key" + SSL_DSN="$SSL_DSN;mysql_ssl_client_cert=/etc/mysql/certs/tls.crt" + SSL_DSN="$SSL_DSN;mysql_ssl_ca_file=/etc/mysql/certs/ca.crt" + if ! pt-show-grants --defaults-file=/etc/mysql/admin_user.cnf $SSL_DSN \ +{{- else }} + if ! pt-show-grants --defaults-file=/etc/mysql/admin_user.cnf \ +{{- end }} + 2>>"$LOG_FILE" > "$TMP_DIR"/grants.sql; then + log ERROR "Failed to create GRANT for all the users" + return 1 + fi + fi + + #Retrieve and create the GRANT files per DB + for db in "${MYSQL_DBNAMES[@]}" + do + echo $($MYSQL --skip-column-names -e "select concat('show grants for ',user,';') \ + from mysql.db where ucase(db)=ucase('$db');") | \ + sed -r "s/show grants for ([a-zA-Z0-9_-]*)/show grants for '\1'/g" | \ + $MYSQL --silent --skip-column-names 2>>$LOG_FILE > $TMP_DIR/${db}_grant.sql + if [ "$?" -eq 0 ] + then + sed -i 's/$/;/' $TMP_DIR/${db}_grant.sql + else + log ERROR "Failed to create GRANT files for ${db}" + return 1 + fi + done + + #Dumping the database + + SQL_FILE=mariadb.$MARIADB_POD_NAMESPACE.${SCOPE} + + $MYSQLDUMP $MYSQL_BACKUP_MYSQLDUMP_OPTIONS "${MYSQL_DBNAMES[@]}" \ + > $TMP_DIR/${SQL_FILE}.sql 2>>$LOG_FILE + if [[ $? -eq 0 && -s $TMP_DIR/${SQL_FILE}.sql ]] + then + log INFO "Database(s) dumped successfully. (SCOPE = ${SCOPE})" + return 0 + else + log ERROR "Backup failed and need attention. (SCOPE = ${SCOPE})" + return 1 + fi +} + +# functions from mariadb-verifier chart + +get_time_delta_secs () { + second_delta=0 + input_date_second=$( date --date="$1" +%s ) + if [ -n "$input_date_second" ]; then + current_date=$( date +"%Y-%m-%dT%H:%M:%SZ" ) + current_date_second=$( date --date="$current_date" +%s ) + ((second_delta=current_date_second-input_date_second)) + if [ "$second_delta" -lt 0 ]; then + second_delta=0 + fi + fi + echo $second_delta +} + + +check_data_freshness () { + archive_file=$(basename "$1") + archive_date=$(echo "$archive_file" | cut -d'.' -f 4) + SCOPE=$2 + + if [[ "${SCOPE}" != "all" ]]; then + log "Data freshness check is skipped for individual database." + return 0 + fi + + log "Checking for data freshness in the backups..." + # Get some idea of which database.table has changed in the last 30m + # Excluding the system DBs and aqua_test_database + # + changed_tables=$(${MYSQL_LIVE} -e "select TABLE_SCHEMA,TABLE_NAME from \ +information_schema.tables where UPDATE_TIME >= SUBTIME(now(),'00:30:00') AND TABLE_SCHEMA \ +NOT IN('information_schema', 'mysql', 'performance_schema', 'sys', 'aqua_test_database');" | \ +awk '{print $1 "." $2}') + + if [ -n "${changed_tables}" ]; then + delta_secs=$(get_time_delta_secs "$archive_date") + age_offset={{ .Values.conf.backup.validateData.ageOffset }} + ((age_threshold=delta_secs+age_offset)) + + data_freshness=false + skipped_freshness=false + + for table in ${changed_tables}; do + tab_schema=$(echo "$table" | awk -F. '{print $1}') + tab_name=$(echo "$table" | awk -F. '{print $2}') + + local_table_existed=$(${MYSQL_LOCAL_SHORT_SILENT} -e "select TABLE_SCHEMA,TABLE_NAME from \ +INFORMATION_SCHEMA.TABLES where TABLE_SCHEMA=\"${tab_schema}\" AND TABLE_NAME=\"${tab_name}\";") + + if [ -n "$local_table_existed" ]; then + # TODO: If last updated field of a table structure has different + # patterns (updated/timstamp), it may be worth to parameterize the patterns. + datetime=$(${MYSQL_LOCAL_SHORT_SILENT} -e "describe ${table};" | \ + awk '(/updated/ || /timestamp/) && /datetime/ {print $1}') + + if [ -n "${datetime}" ]; then + data_ages=$(${MYSQL_LOCAL_SHORT_SILENT} -e "select \ +time_to_sec(timediff(now(),${datetime})) from ${table} where ${datetime} is not null order by 1 limit 10;") + + for age in $data_ages; do + if [ "$age" -le $age_threshold ]; then + data_freshness=true + break + fi + done + + # As long as there is an indication of data freshness, no need to check further + if [ "$data_freshness" = true ] ; then + break + fi + else + skipped_freshness=true + log "No indicator to determine data freshness for table $table. Skipped data freshness check." + + # Dumping out table structure to determine if enhancement is needed to include this table + debug_info=$(${MYSQL_LOCAL} --skip-column-names -e "describe ${table};" | awk '{print $2 " " $1}') + log "$debug_info" "DEBUG" + fi + else + log "Table $table doesn't exist in local database" + skipped_freshness=true + fi + done + + if [ "$data_freshness" = true ] ; then + log "Database passed integrity (data freshness) check." + else + if [ "$skipped_freshness" = false ] ; then + log "Local backup database restore failed integrity check." "ERROR" + log "The backup may not have captured the up-to-date data." "INFO" + return 1 + fi + fi + else + log "No tables changed in this backup. Skipped data freshness check as the" + log "check should have been performed by previous validation runs." + fi + + return 0 +} + + +cleanup_local_databases () { + old_local_dbs=$(${MYSQL_LOCAL_SHORT_SILENT} -e 'show databases;' | \ + grep -ivE 'information_schema|performance_schema|mysql|sys' || true) + + for db in $old_local_dbs; do + ${MYSQL_LOCAL_SHORT_SILENT} -e "drop database $db;" + done +} + +list_archive_dir () { + archive_dir_content=$(ls -1R "$ARCHIVE_DIR") + if [ -n "$archive_dir_content" ]; then + log "Content of $ARCHIVE_DIR" + log "${archive_dir_content}" + fi +} + +remove_remote_archive_file () { + archive_file=$(basename "$1") + token_req_file=$(mktemp --suffix ".json") + header_file=$(mktemp) + resp_file=$(mktemp --suffix ".json") + http_resp="404" + + HEADER_CONTENT_TYPE="Content-Type: application/json" + HEADER_ACCEPT="Accept: application/json" + + cat << JSON_EOF > "$token_req_file" +{ + "auth": { + "identity": { + "methods": [ + "password" + ], + "password": { + "user": { + "domain": { + "name": "${OS_USER_DOMAIN_NAME}" + }, + "name": "${OS_USERNAME}", + "password": "${OS_PASSWORD}" + } + } + }, + "scope": { + "project": { + "domain": { + "name": "${OS_PROJECT_DOMAIN_NAME}" + }, + "name": "${OS_PROJECT_NAME}" + } + } + } +} +JSON_EOF + + http_resp=$(curl -s -X POST "$OS_AUTH_URL/auth/tokens" -H "${HEADER_CONTENT_TYPE}" \ + -H "${HEADER_ACCEPT}" -d @"${token_req_file}" -D "$header_file" -o "$resp_file" -w "%{http_code}") + + if [ "$http_resp" = "201" ]; then + OS_TOKEN=$(grep -i "x-subject-token" "$header_file" | cut -d' ' -f2 | tr -d "\r") + + if [ -n "$OS_TOKEN" ]; then + OS_OBJ_URL=$(python3 -c "import json,sys;print([[ep['url'] for ep in obj['endpoints'] if ep['interface']=='public'] for obj in json.load(sys.stdin)['token']['catalog'] if obj['type']=='object-store'][0][0])" < "$resp_file") + + if [ -n "$OS_OBJ_URL" ]; then + http_resp=$(curl -s -X DELETE "$OS_OBJ_URL/$CONTAINER_NAME/$archive_file" \ + -H "${HEADER_CONTENT_TYPE}" -H "${HEADER_ACCEPT}" \ + -H "X-Auth-Token: ${OS_TOKEN}" -D "$header_file" -o "$resp_file" -w "%{http_code}") + fi + fi + fi + + if [ "$http_resp" == "404" ] ; then + log "Failed to cleanup remote backup. Container object $archive_file is not on RGW." + return 1 + fi + + if [ "$http_resp" != "204" ] ; then + log "Failed to cleanup remote backup. Cannot delete container object $archive_file" "ERROR" + cat "$header_file" + cat "$resp_file" + fi + return 0 +} + +handle_bad_archive_file () { + archive_file=$1 + + if [ ! -d "$BAD_ARCHIVE_DIR" ]; then + mkdir -p "$BAD_ARCHIVE_DIR" + fi + + # Move the file to quarantine directory such that + # file won't be used for restore in case of recovery + # + log "Moving $i to $BAD_ARCHIVE_DIR..." + mv "$i" "$BAD_ARCHIVE_DIR" + log "Removing $i from remote RGW..." + if remove_remote_archive_file "$i"; then + log "File $i has been successfully removed from RGW." + else + log "FIle $i cannot be removed form RGW." "ERROR" + return 1 + fi + + # Atmost only three bad files are kept. Deleting the oldest if + # number of files exceeded the threshold. + # + bad_files=$(find "$BAD_ARCHIVE_DIR" -name "*.tar.gz" 2>/dev/null | wc -l) + if [ "$bad_files" -gt 3 ]; then + ((bad_files=bad_files-3)) + delete_files=$(find "$BAD_ARCHIVE_DIR" -name "*.tar.gz" 2>/dev/null | sort | head --lines=$bad_files) + for b in $delete_files; do + log "Deleting $b..." + rm -f "${b}" + done + fi + return 0 +} + +cleanup_old_validation_result_file () { + clean_files=$(find "$ARCHIVE_DIR" -maxdepth 1 -name "*.passed" 2>/dev/null) + for d in $clean_files; do + archive_file=${d/.passed} + if [ ! -f "$archive_file" ]; then + log "Deleting $d as its associated archive file $archive_file nolonger existed." + rm -f "${d}" + fi + done +} + +validate_databases_backup () { + archive_file=$1 + SCOPE=${2:-"all"} + + restore_log='/tmp/restore_error.log' + tmp_dir=$(mktemp -d) + + rm -f $restore_log + cd "$tmp_dir" + log "Decompressing archive $archive_file..." + if ! tar zxvf - < "$archive_file" 1>/dev/null; then + log "Database restore from local backup failed. Archive decompression failed." "ERROR" + return 1 + fi + + db_list_file="$tmp_dir/db.list" + if [[ -e "$db_list_file" ]]; then + dbs=$(sort < "$db_list_file" | grep -ivE sys | tr '\n' ' ') + else + dbs=" " + fi + + sql_file="${tmp_dir}/mariadb.${MARIADB_POD_NAMESPACE}.${SCOPE}.sql" + + if [[ "${SCOPE}" == "all" ]]; then + grant_file="${tmp_dir}/grants.sql" + else + grant_file="${tmp_dir}/${SCOPE}_grant.sql" + fi + + if [[ -f $sql_file ]]; then + if $MYSQL_LOCAL < "$sql_file" 2>$restore_log; then + local_dbs=$(${MYSQL_LOCAL_SHORT_SILENT} -e 'show databases;' | \ + grep -ivE 'information_schema|performance_schema|mysql|sys' | sort | tr '\n' ' ') + + if [ "$dbs" = "$local_dbs" ]; then + log "Databases restored successful." + else + log "Database restore from local backup failed. Database mismatched between local backup and local server" "ERROR" + log "Databases restored on local server: $local_dbs" "DEBUG" + log "Databases in the local backup: $dbs" "DEBUG" + return 1 + fi + else + log "Database restore from local backup failed. $dbs" "ERROR" + cat $restore_log + return 1 + fi + + if [[ -f $grant_file ]]; then + if $MYSQL_LOCAL < "$grant_file" 2>$restore_log; then + if ! $MYSQL_LOCAL -e 'flush privileges;'; then + log "Database restore from local backup failed. Failed to flush privileges." "ERROR" + return 1 + fi + log "Databases permission restored successful." + else + log "Database restore from local backup failed. Databases permission failed to restore." "ERROR" + cat "$restore_log" + cat "$grant_file" + log "Local DBs: $local_dbs" "DEBUG" + return 1 + fi + else + log "Database restore from local backup failed. There is no permission file available" "ERROR" + return 1 + fi + + if ! check_data_freshness "$archive_file" ${SCOPE}; then + # Log has already generated during check data freshness + return 1 + fi + else + log "Database restore from local backup failed. There is no database file available to restore from" "ERROR" + return 1 + fi + + return 0 +} + +# end of functions form mariadb verifier chart + +# Verify all the databases backup archives +verify_databases_backup_archives() { + SCOPE=${1:-"all"} + + # verification code + export DB_NAME="mariadb" + export ARCHIVE_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${MARIADB_POD_NAMESPACE}/${DB_NAME}/archive + export BAD_ARCHIVE_DIR=${ARCHIVE_DIR}/quarantine + export MYSQL_OPTS="--silent --skip-column-names" + export MYSQL_LIVE="mysql ${MYSQL_OPTS}" + export MYSQL_LOCAL_OPTS="" + export MYSQL_LOCAL_SHORT="mysql ${MYSQL_LOCAL_OPTS} --connect-timeout 2" + export MYSQL_LOCAL_SHORT_SILENT="${MYSQL_LOCAL_SHORT} ${MYSQL_OPTS}" + export MYSQL_LOCAL="mysql ${MYSQL_LOCAL_OPTS} --connect-timeout 10" + + max_wait={{ .Values.conf.mariadb_server.setup_wait.iteration }} + duration={{ .Values.conf.mariadb_server.setup_wait.duration }} + counter=0 + dbisup=false + + log "Waiting for Mariadb backup verification server to start..." + + # During Mariadb init/startup process, a temporary server is startup + # and shutdown prior to starting up the normal server. + # To avoid prematurely determine server availability, lets snooze + # a bit to give time for the process to complete prior to issue + # mysql commands. + # + + + while [ $counter -lt $max_wait ]; do + if ! $MYSQL_LOCAL_SHORT -e 'select 1' > /dev/null 2>&1 ; then + sleep $duration + ((counter=counter+1)) + else + # Lets sleep for an additional duration just in case async + # init takes a bit more time to complete. + # + sleep $duration + dbisup=true + counter=$max_wait + fi + done + + if ! $dbisup; then + log "Mariadb backup verification server is not running" "ERROR" + return 1 + fi + + # During Mariadb init process, a test database will be briefly + # created and deleted. Adding to the exclusion list for some + # edge cases + # + clean_db=$(${MYSQL_LOCAL_SHORT_SILENT} -e 'show databases;' | \ + grep -ivE 'information_schema|performance_schema|mysql|test|sys' || true) + + if [[ -z "${clean_db// }" ]]; then + log "Clean Server is up and running" + else + cleanup_local_databases + log "Old databases found on the Mariadb backup verification server were cleaned." + clean_db=$(${MYSQL_LOCAL_SHORT_SILENT} -e 'show databases;' | \ + grep -ivE 'information_schema|performance_schema|mysql|test|sys' || true) + + if [[ -z "${clean_db// }" ]]; then + log "Clean Server is up and running" + else + log "Cannot clean old databases on verification server." "ERROR" + return 1 + fi + log "The server is ready for verification." + fi + + # Starting with 10.4.13, new definer mariadb.sys was added. However, mariadb.sys was deleted + # during init mariadb as it was not on the exclusion list. This corrupted the view of mysql.user. + # Insert the tuple back to avoid other similar issues with error i.e + # The user specified as a definer ('mariadb.sys'@'localhost') does not exist + # + # Before insert the tuple mentioned above, we should make sure that the MariaDB version is 10.4.+ + mariadb_version=$($MYSQL_LOCAL_SHORT -e "status" | grep -E '^Server\s+version:') + log "Current database ${mariadb_version}" + if [[ ! -z ${mariadb_version} && -z $(grep '10.2' <<< ${mariadb_version}}) ]]; then + if [[ -z $(grep 'mariadb.sys' <<< $($MYSQL_LOCAL_SHORT mysql -e "select * from global_priv where user='mariadb.sys'")) ]]; then + $MYSQL_LOCAL_SHORT -e "insert into mysql.global_priv values ('localhost','mariadb.sys',\ + '{\"access\":0,\"plugin\":\"mysql_native_password\",\"authentication_string\":\"\",\"account_locked\":true,\"password_last_changed\":0}');" + $MYSQL_LOCAL_SHORT -e 'flush privileges;' + fi + fi + + # Ensure archive dir existed + if [ -d "$ARCHIVE_DIR" ]; then + # List archive dir before + list_archive_dir + + # Ensure the local databases are clean for each restore validation + # + cleanup_local_databases + + if [[ "${SCOPE}" == "all" ]]; then + archive_files=$(find "$ARCHIVE_DIR" -maxdepth 1 -name "*.tar.gz" 2>/dev/null | sort) + for i in $archive_files; do + archive_file_passed=$i.passed + if [ ! -f "$archive_file_passed" ]; then + log "Validating archive file $i..." + if validate_databases_backup "$i"; then + touch "$archive_file_passed" + else + if handle_bad_archive_file "$i"; then + log "File $i has been removed from RGW." + else + log "File $i cannot be removed from RGW." "ERROR" + return 1 + fi + fi + fi + done + else + archive_files=$(find "$ARCHIVE_DIR" -maxdepth 1 -name "*.tar.gz" 2>/dev/null | grep "${SCOPE}" | sort) + for i in $archive_files; do + archive_file_passed=$i.passed + if [ ! -f "$archive_file_passed" ]; then + log "Validating archive file $i..." + if validate_databases_backup "${i}" "${SCOPE}"; then + touch "$archive_file_passed" + else + if handle_bad_archive_file "$i"; then + log "File $i has been removed from RGW." + else + log "File $i cannot be removed from RGW." "ERROR" + return 1 + fi + fi + fi + done + fi + + + # Cleanup passed files if its archive file nolonger existed + cleanup_old_validation_result_file + + # List archive dir after + list_archive_dir + fi + + + return 0 +} + +# Call main program to start the database backup +backup_databases ${SCOPE} diff --git a/mariadb/templates/bin/_health.sh.tpl b/mariadb/templates/bin/_health.sh.tpl new file mode 100644 index 0000000000..fb4be06456 --- /dev/null +++ b/mariadb/templates/bin/_health.sh.tpl @@ -0,0 +1,139 @@ +#!/usr/bin/env bash + +########################################################################### +# Copyright 2017 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +######################################################################### + +set -e + +MYSQL="mysql \ + --defaults-file=/etc/mysql/admin_user.cnf \ + --host=localhost \ +{{- if .Values.manifests.certificates }} + --ssl-verify-server-cert=false \ + --ssl-ca=/etc/mysql/certs/ca.crt \ + --ssl-key=/etc/mysql/certs/tls.key \ + --ssl-cert=/etc/mysql/certs/tls.crt \ +{{- end }} + --connect-timeout 2" + +mysql_query () { + TABLE=$1 + KEY=$2 + $MYSQL -e "show ${TABLE} like \"${KEY}\"" | \ + awk "/${KEY}/ { print \$NF; exit }" +} + +function usage { + echo "Usage: $0 [-t ] [-d ]" 1>&2 + exit 1 +} + +PROBE_TYPE='' + +while getopts ":t:d:" opt; do + case $opt in + t) + PROBE_TYPE=$OPTARG + ;; + d) + DISK_ALARM_LIMIT=$OPTARG + ;; + *) + usage + ;; + esac +done +shift $((OPTIND-1)) + +check_readiness () { + if ! $MYSQL -e 'select 1' > /dev/null 2>&1 ; then + echo "Select from mysql failed" + exit 1 + fi + + DATADIR=$(mysql_query variables datadir) + TMPDIR=$(mysql_query variables tmpdir) + for partition in ${DATADIR} ${TMPDIR}; do + if [ "$(df --output=pcent ${partition} | grep -Po '\d+')" -ge "${DISK_ALARM_LIMIT:-100}" ]; then + echo "[ALARM] Critical high disk space utilization of ${partition}" + exit 1 + fi + done + + if [ "x$(mysql_query status wsrep_ready)" != "xON" ]; then + echo "WSREP says the node can not receive queries" + exit 1 + fi + if [ "x$(mysql_query status wsrep_connected)" != "xON" ]; then + echo "WSREP not connected" + exit 1 + fi + if [ "x$(mysql_query status wsrep_cluster_status)" != "xPrimary" ]; then + echo "Not in primary cluster" + exit 1 + fi + if [ "x$(mysql_query status wsrep_local_state_comment)" != "xSynced" ]; then + echo "WSREP not synced" + exit 1 + fi +} + +check_liveness () { + if pidof mysql_upgrade > /dev/null 2>&1 ; then + echo "The process mysql_upgrade is active. Skip rest checks" + exit 0 + fi + if ! pidof mysqld > /dev/null 2>&1 ; then + echo "The mysqld pid not found" + exit 1 + fi + # NOTE(mkarpin): SST process may take significant time in case of large databases, + # killing mysqld during SST may destroy all data on the node. + local datadir="/var/lib/mysql" + if [ -f ${datadir}/sst_in_progress ]; then + echo "SST is still in progress, skip further checks as mysql won't respond" + else + # NOTE(vsaienko): in some cases maria might stuck during IST, or when neighbours + # IPs are changed. Here we check that we can connect to mysql socket to ensure + # process is alive. + if ! $MYSQL -e "show status like 'wsrep_cluster_status'" > /dev/null 2>&1 ; then + echo "Can't connect to mysql socket" + exit 1 + fi + # Detect node that is not connected to wsrep provider + if [ "x$(mysql_query status wsrep_ready)" != "xON" ]; then + echo "WSREP says the node can not receive queries" + exit 1 + fi + if [ "x$(mysql_query status wsrep_connected)" != "xON" ]; then + echo "WSREP not connected" + exit 1 + fi + fi +} + +case $PROBE_TYPE in + liveness) + check_liveness + ;; + readiness) + check_readiness + ;; + *) + echo "Unknown probe type: ${PROBE_TYPE}" + usage + ;; +esac diff --git a/mariadb/templates/bin/_mariadb-wait-for-cluster.py.tpl b/mariadb/templates/bin/_mariadb-wait-for-cluster.py.tpl new file mode 100644 index 0000000000..c1dbfeeeb9 --- /dev/null +++ b/mariadb/templates/bin/_mariadb-wait-for-cluster.py.tpl @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 + +import datetime +from enum import Enum +import logging +import os +import sys +import time + +import pymysql +import pykube + +MARIADB_HOST = os.getenv("MARIADB_HOST") +MARIADB_PASSWORD = os.getenv("MARIADB_PASSWORD") +MARIADB_REPLICAS = os.getenv("MARIADB_REPLICAS") + +MARIADB_CLUSTER_STATE_LOG_LEVEL = os.getenv("MARIADB_CLUSTER_STATE_LOG_LEVEL", "INFO") + +MARIADB_CLUSTER_STABILITY_COUNT = int( + os.getenv("MARIADB_CLUSTER_STABILITY_COUNT", "30") +) +MARIADB_CLUSTER_STABILITY_WAIT = int(os.getenv("MARIADB_CLUSTER_STABILITY_WAIT", "4")) +MARIADB_CLUSTER_CHECK_WAIT = int(os.getenv("MARIADB_CLUSTER_CHECK_WAIT", "30")) + +MARIADB_CLUSTER_STATE_CONFIGMAP = os.getenv("MARIADB_CLUSTER_STATE_CONFIGMAP") +MARIADB_CLUSTER_STATE_CONFIGMAP_NAMESPACE = os.getenv( + "MARIADB_CLUSTER_STATE_CONFIGMAP_NAMESPACE", "openstack" +) +MARIADB_CLUSTER_STATE_PYKUBE_REQUEST_TIMEOUT = int( + os.getenv("MARIADB_CLUSTER_STATE_PYKUBE_REQUEST_TIMEOUT", 60) +) + +log_level = MARIADB_CLUSTER_STATE_LOG_LEVEL +logging.basicConfig( + stream=sys.stdout, + format="%(asctime)s %(levelname)s %(name)s %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", +) +LOG = logging.getLogger("mariadb-cluster-wait") +LOG.setLevel(log_level) + + +def login(): + config = pykube.KubeConfig.from_env() + client = pykube.HTTPClient( + config=config, timeout=MARIADB_CLUSTER_STATE_PYKUBE_REQUEST_TIMEOUT + ) + LOG.info(f"Created k8s api client from context {config.current_context}") + return client + + +api = login() +cluster_state_map = ( + pykube.ConfigMap.objects(api) + .filter(namespace=MARIADB_CLUSTER_STATE_CONFIGMAP_NAMESPACE) + .get_by_name(MARIADB_CLUSTER_STATE_CONFIGMAP) +) + + +def get_current_state(cluster_state_map): + cluster_state_map.get( + MARIADB_CLUSTER_STATE_INITIAL_BOOTSTRAP_COMPLETED_KEY, "False" + ) + + +def retry(times, exceptions): + def decorator(func): + def newfn(*args, **kwargs): + attempt = 0 + while attempt < times: + try: + return func(*args, **kwargs) + except exceptions: + attempt += 1 + LOG.exception( + f"Exception thrown when attempting to run {func}, attempt {attempt} of {times}" + ) + return func(*args, **kwargs) + return newfn + return decorator + + +class initalClusterState: + + initial_state_key = "initial-bootstrap-completed.cluster" + + @retry(times=100, exceptions=(Exception)) + def __init__(self, api, namespace, name): + self.namespace = namespace + self.name = name + self.cm = ( + pykube.ConfigMap.objects(api) + .filter(namespace=self.namespace) + .get_by_name(self.name) + ) + + def get_default(self): + """We have deployments with completed job, but it is not reflected + in the configmap state. Assume when configmap is created more than + 1h and we doing update/restart, and key not in map this is + existed environment. So we assume the cluster was initialy bootstrapped. + This is needed to avoid manual actions. + """ + now = datetime.datetime.utcnow() + created_at = datetime.datetime.strptime( + self.cm.obj["metadata"]["creationTimestamp"], "%Y-%m-%dT%H:%M:%SZ" + ) + delta = datetime.timedelta(seconds=3600) + + if now - created_at > delta: + self.complete() + return "COMPLETED" + return "NOT_COMPLETED" + + @property + @retry(times=10, exceptions=(Exception)) + def is_completed(self): + + self.cm.reload() + if self.initial_state_key in self.cm.obj["data"]: + return self.cm.obj["data"][self.initial_state_key] + + return self.get_default() == "COMPLETED" + + @retry(times=100, exceptions=(Exception)) + def complete(self): + patch = {"data": {self.initial_state_key: "COMPLETED"}} + self.cm.patch(patch) + + +ics = initalClusterState( + api, MARIADB_CLUSTER_STATE_CONFIGMAP_NAMESPACE, MARIADB_CLUSTER_STATE_CONFIGMAP +) + +if ics.is_completed: + LOG.info("The initial bootstrap was completed, skipping wait...") + sys.exit(0) + +LOG.info("Checking for mariadb cluster state.") + + +def is_mariadb_stabe(): + try: + wsrep_OK = { + "wsrep_ready": "ON", + "wsrep_connected": "ON", + "wsrep_cluster_status": "Primary", + "wsrep_local_state_comment": "Synced", + "wsrep_cluster_size": str(MARIADB_REPLICAS), + } + wsrep_vars = ",".join(["'" + var + "'" for var in wsrep_OK.keys()]) + db_cursor = pymysql.connect( + host=MARIADB_HOST, password=MARIADB_PASSWORD, + read_default_file="/etc/mysql/admin_user.cnf" + ).cursor() + db_cursor.execute(f"SHOW GLOBAL STATUS WHERE Variable_name IN ({wsrep_vars})") + wsrep_vars = db_cursor.fetchall() + diff = set(wsrep_vars).difference(set(wsrep_OK.items())) + if diff: + LOG.error(f"The wsrep is not OK: {diff}") + else: + LOG.info("The wspep is ready") + return True + except Exception as e: + LOG.exception(f"Got exception while checking state. {e}") + return False + + +count = 0 +ready = False +stable_for = 1 + +while True: + if is_mariadb_stabe(): + stable_for += 1 + LOG.info( + f"The cluster is stable for {stable_for} out of {MARIADB_CLUSTER_STABILITY_COUNT}" + ) + if stable_for == MARIADB_CLUSTER_STABILITY_COUNT: + ics.complete() + sys.exit(0) + else: + LOG.info(f"Sleeping for {MARIADB_CLUSTER_STABILITY_WAIT}") + time.sleep(MARIADB_CLUSTER_STABILITY_WAIT) + continue + else: + LOG.info("Resetting stable_for count.") + stable_for = 0 + LOG.info(f"Sleeping for {MARIADB_CLUSTER_CHECK_WAIT}") + time.sleep(MARIADB_CLUSTER_CHECK_WAIT) diff --git a/mariadb/templates/bin/_mariadb_controller.py.tpl b/mariadb/templates/bin/_mariadb_controller.py.tpl new file mode 100644 index 0000000000..faf5195a53 --- /dev/null +++ b/mariadb/templates/bin/_mariadb_controller.py.tpl @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 + +""" +Mariadb controller + +The script is responsible for set mariadb_role: primary to first +active pod in mariadb deployment. + +Env variables: +MARIADB_CONTROLLER_DEBUG: Flag to enable debug when set to 1. +MARIADB_CONTROLLER_CHECK_PODS_DELAY: The delay between check pod attempts. +MARIADB_CONTROLLER_PYKUBE_REQUEST_TIMEOUT: The timeout for kubernetes http session +MARIADB_CONTROLLER_PODS_NAMESPACE: The namespace to look for mariadb pods. +MARIADB_MASTER_SERVICE_NAME: The name of master service for mariadb. + +Changelog: +0.1.0: Initial varsion +""" + + +import logging +import os +import sys +import time + +import pykube + +MARIADB_CONTROLLER_DEBUG = os.getenv("MARIADB_CONTROLLER_DEBUG") +MARIADB_CONTROLLER_CHECK_PODS_DELAY = int( + os.getenv("MARIADB_CONTROLLER_CHECK_PODS_DELAY", 10) +) +MARIADB_CONTROLLER_PYKUBE_REQUEST_TIMEOUT = int( + os.getenv("MARIADB_CONTROLLER_PYKUBE_REQUEST_TIMEOUT", 60) +) +MARIADB_CONTROLLER_PODS_NAMESPACE = os.getenv( + "MARIADB_CONTROLLER_PODS_NAMESPACE", "openstack" +) +MARIADB_MASTER_SERVICE_NAME = os.getenv( + "MARIADB_MASTER_SERVICE_NAME", "mariadb" +) + +log_level = "DEBUG" if MARIADB_CONTROLLER_DEBUG else "INFO" +logging.basicConfig( + stream=sys.stdout, + format="%(asctime)s %(levelname)s %(name)s %(message)s", + datefmt="%Y-%m-%d %H:%M:%S", +) +LOG = logging.getLogger("mariadb-controller") + +LOG.setLevel(log_level) + + +def login(): + config = pykube.KubeConfig.from_env() + client = pykube.HTTPClient( + config=config, timeout=MARIADB_CONTROLLER_PYKUBE_REQUEST_TIMEOUT + ) + LOG.info(f"Created k8s api client from context {config.current_context}") + return client + + +api = login() + + +def resource_list(klass, selector, namespace=None): + return klass.objects(api).filter(namespace=namespace, selector=selector) + + +def get_mariadb_pods(): + sorted_pods = sorted( + resource_list( + pykube.Pod, + {"application": "mariadb", "component": "server"}, + MARIADB_CONTROLLER_PODS_NAMESPACE, + ).iterator(), + key=lambda i: i.name, + ) + return sorted_pods + + +def get_mariadb_master_service(namespace): + return pykube.Service.objects(api).filter(namespace=namespace).get(name=MARIADB_MASTER_SERVICE_NAME) + + +def link_master_service(pod): + svc = get_mariadb_master_service(MARIADB_CONTROLLER_PODS_NAMESPACE) + svc.reload() + if svc.obj['spec']['selector'].get('statefulset.kubernetes.io/pod-name') == pod.name: + LOG.debug(f"Nothing to do, master service points to {pod.name}") + else: + svc.obj['spec']['selector']['statefulset.kubernetes.io/pod-name'] = pod.name + svc.update() + LOG.info(f"Link master service with {pod.name}") + + +def is_ready(pod): + if pod.ready and "deletionTimestamp" not in pod.metadata: + return True + + +def main(): + while True: + for pod in get_mariadb_pods(): + pod.reload() + if is_ready(pod): + link_master_service(pod) + break + LOG.debug(f"Sleeping for {MARIADB_CONTROLLER_CHECK_PODS_DELAY}") + time.sleep(MARIADB_CONTROLLER_CHECK_PODS_DELAY) + + +main() diff --git a/mariadb/templates/bin/_prometheus-create-mysql-user.sh.tpl b/mariadb/templates/bin/_prometheus-create-mysql-user.sh.tpl new file mode 100644 index 0000000000..e1355fe62b --- /dev/null +++ b/mariadb/templates/bin/_prometheus-create-mysql-user.sh.tpl @@ -0,0 +1,50 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -e + + # SLAVE MONITOR + # Grants ability to SHOW SLAVE STATUS, SHOW REPLICA STATUS, + # SHOW ALL SLAVES STATUS, SHOW ALL REPLICAS STATUS, SHOW RELAYLOG EVENTS. + # New privilege added in MariaDB Enterprise Server 10.5.8-5. Alias for REPLICA MONITOR. + # + # REPLICATION CLIENT + # Grants ability to SHOW MASTER STATUS, SHOW SLAVE STATUS, SHOW BINARY LOGS. In ES10.5, + # is an alias for BINLOG MONITOR and the capabilities have changed. BINLOG MONITOR grants + # ability to SHOW MASTER STATUS, SHOW BINARY LOGS, SHOW BINLOG EVENTS, and SHOW BINLOG STATUS. + + mariadb_version=$(mysql --defaults-file=/etc/mysql/admin_user.cnf -e "status" | grep -E '^Server\s+version:') + echo "Current database ${mariadb_version}" + + if [[ ! -z ${mariadb_version} && -z $(grep -E '10.2|10.3|10.4' <<< ${mariadb_version}) ]]; then + # In case MariaDB version is 10.2.x-10.4.x - we use old privileges definitions + if ! mysql --defaults-file=/etc/mysql/admin_user.cnf -e \ + "CREATE OR REPLACE USER '${EXPORTER_USER}'@'%' IDENTIFIED BY '${EXPORTER_PASSWORD}'; \ + GRANT SLAVE MONITOR, PROCESS, BINLOG MONITOR, SLAVE MONITOR, SELECT ON *.* TO '${EXPORTER_USER}'@'%' ${MARIADB_X509}; \ + FLUSH PRIVILEGES;" ; then + echo "ERROR: Could not create user: ${EXPORTER_USER}" + exit 1 + fi + else + # here we use new MariaDB privileges definitions defines since version 10.5 + if ! mysql --defaults-file=/etc/mysql/admin_user.cnf -e \ + "CREATE OR REPLACE USER '${EXPORTER_USER}'@'%' IDENTIFIED BY '${EXPORTER_PASSWORD}'; \ + GRANT SLAVE MONITOR, PROCESS, REPLICATION CLIENT, SELECT ON *.* TO '${EXPORTER_USER}'@'%' ${MARIADB_X509}; \ + FLUSH PRIVILEGES;" ; then + echo "ERROR: Could not create user: ${EXPORTER_USER}" + exit 1 + fi + fi diff --git a/mariadb/templates/bin/_prometheus-mysqld-exporter.sh.tpl b/mariadb/templates/bin/_prometheus-mysqld-exporter.sh.tpl new file mode 100644 index 0000000000..d794be3749 --- /dev/null +++ b/mariadb/templates/bin/_prometheus-mysqld-exporter.sh.tpl @@ -0,0 +1,57 @@ +#!/bin/sh + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +compareVersions() { +echo $1 $2 | \ +awk '{ split($1, a, "."); + split($2, b, "."); + res = -1; + for (i = 1; i <= 3; i++){ + if (a[i] < b[i]) { + res =-1; + break; + } else if (a[i] > b[i]) { + res = 1; + break; + } else if (a[i] == b[i]) { + if (i == 3) { + res = 0; + break; + } else { + continue; + } + } + } + print res; + }' +} + +MYSQL_EXPORTER_VER=`/bin/mysqld_exporter --version 2>&1 | grep "mysqld_exporter" | awk '{print $3}'` + +#in versions greater than 0.10.0 different configuration flags are used: +#https://github.com/prometheus/mysqld_exporter/commit/66c41ac7eb90a74518a6ecf6c6bb06464eb68db8 +compverResult=`compareVersions "${MYSQL_EXPORTER_VER}" "0.10.0"` +CONFIG_FLAG_PREFIX='-' +if [ ${compverResult} -gt 0 ]; then + CONFIG_FLAG_PREFIX='--' +fi + +exec /bin/mysqld_exporter \ + ${CONFIG_FLAG_PREFIX}config.my-cnf=/etc/mysql/mysql_user.cnf \ + ${CONFIG_FLAG_PREFIX}web.listen-address="${POD_IP}:${LISTEN_PORT}" \ + ${CONFIG_FLAG_PREFIX}web.telemetry-path="$TELEMETRY_PATH" diff --git a/mariadb/templates/bin/_restore_mariadb.sh.tpl b/mariadb/templates/bin/_restore_mariadb.sh.tpl new file mode 100755 index 0000000000..334ba85bc6 --- /dev/null +++ b/mariadb/templates/bin/_restore_mariadb.sh.tpl @@ -0,0 +1,328 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +{{- $envAll := . }} + +# Capture the user's command line arguments +ARGS=("$@") + +if [[ -s /tmp/restore_main.sh ]]; then + source /tmp/restore_main.sh +else + echo "File /tmp/restore_main.sh does not exist." + exit 1 +fi + +# Export the variables needed by the framework +export DB_NAME="mariadb" +export DB_NAMESPACE=${MARIADB_POD_NAMESPACE} +export ARCHIVE_DIR=${MARIADB_BACKUP_BASE_DIR}/db/${DB_NAMESPACE}/${DB_NAME}/archive + +RESTORE_USER='restoreuser' +RESTORE_PW=$(pwgen 16 1) +RESTORE_LOG='/tmp/restore_error.log' +rm -f $RESTORE_LOG + +# This is for commands which require admin access +MYSQL="mysql \ + --defaults-file=/etc/mysql/admin_user.cnf \ + --host=$MARIADB_SERVER_SERVICE_HOST \ + --connect-timeout 10" + +# This is for commands which we want the temporary "restore" user +# to execute +RESTORE_CMD="mysql \ + --user=${RESTORE_USER} \ + --password=${RESTORE_PW} \ + --host=$MARIADB_SERVER_SERVICE_HOST \ +{{- if .Values.manifests.certificates }} + --ssl-ca=/etc/mysql/certs/ca.crt \ + --ssl-key=/etc/mysql/certs/tls.key \ + --ssl-cert=/etc/mysql/certs/tls.crt \ +{{- end }} + --connect-timeout 10" + +# Get a single database data from the SQL file. +# $1 - database name +# $2 - sql file path +current_db_desc() { + PATTERN="-- Current Database:" + sed -n "/${PATTERN} \`$1\`/,/${PATTERN}/p" $2 +} + +#Return all database from an archive +get_databases() { + TMP_DIR=$1 + DB_FILE=$2 + + if [[ -e ${TMP_DIR}/db.list ]] + then + DBS=$(cat ${TMP_DIR}/db.list | \ + grep -ivE 'information_schema|performance_schema|mysql|sys' ) + else + DBS=" " + fi + + echo $DBS > $DB_FILE +} + +# Determine sql file from 2 options - current and legacy one +# if current is not found check that there is no other namespaced dump file +# before falling back to legacy one +_get_sql_file() { + TMP_DIR=$1 + SQL_FILE="${TMP_DIR}/mariadb.${MARIADB_POD_NAMESPACE}.*.sql" + LEGACY_SQL_FILE="${TMP_DIR}/mariadb.*.sql" + INVALID_SQL_FILE="${TMP_DIR}/mariadb.*.*.sql" + if [ -f ${SQL_FILE} ] + then + echo "Found $(ls ${SQL_FILE})" > /dev/stderr + printf ${SQL_FILE} + elif [ -f ${INVALID_SQL_FILE} ] + then + echo "Expected to find ${SQL_FILE} or ${LEGACY_SQL_FILE}, but found $(ls ${INVALID_SQL_FILE})" > /dev/stderr + elif [ -f ${LEGACY_SQL_FILE} ] + then + echo "Falling back to legacy naming ${LEGACY_SQL_FILE}. Found $(ls ${LEGACY_SQL_FILE})" > /dev/stderr + printf ${LEGACY_SQL_FILE} + fi +} + +# Extract all tables of a database from an archive and put them in the requested +# file. +get_tables() { + DATABASE=$1 + TMP_DIR=$2 + TABLE_FILE=$3 + + SQL_FILE=$(_get_sql_file $TMP_DIR) + if [ ! -z $SQL_FILE ]; then + current_db_desc ${DATABASE} ${SQL_FILE} \ + | grep "^CREATE TABLE" | awk -F '`' '{print $2}' \ + > $TABLE_FILE + else + # Error, cannot report the tables + echo "No SQL file found - cannot extract the tables" + return 1 + fi +} + +# Extract all rows in the given table of a database from an archive and put +# them in the requested file. +get_rows() { + DATABASE=$1 + TABLE=$2 + TMP_DIR=$3 + ROW_FILE=$4 + + SQL_FILE=$(_get_sql_file $TMP_DIR) + if [ ! -z $SQL_FILE ]; then + current_db_desc ${DATABASE} ${SQL_FILE} \ + | grep "INSERT INTO \`${TABLE}\` VALUES" > $ROW_FILE + return 0 + else + # Error, cannot report the rows + echo "No SQL file found - cannot extract the rows" + return 1 + fi +} + +# Extract the schema for the given table in the given database belonging to +# the archive file found in the TMP_DIR. +get_schema() { + DATABASE=$1 + TABLE=$2 + TMP_DIR=$3 + SCHEMA_FILE=$4 + + SQL_FILE=$(_get_sql_file $TMP_DIR) + if [ ! -z $SQL_FILE ]; then + DB_FILE=$(mktemp -p /tmp) + current_db_desc ${DATABASE} ${SQL_FILE} > ${DB_FILE} + sed -n /'CREATE TABLE `'$TABLE'`'/,/'--'/p ${DB_FILE} > ${SCHEMA_FILE} + if [[ ! (-s ${SCHEMA_FILE}) ]]; then + sed -n /'CREATE TABLE IF NOT EXISTS `'$TABLE'`'/,/'--'/p ${DB_FILE} \ + > ${SCHEMA_FILE} + fi + rm -f ${DB_FILE} + else + # Error, cannot report the rows + echo "No SQL file found - cannot extract the schema" + return 1 + fi +} + +# Create temporary user for restoring specific databases. +create_restore_user() { + restore_db=$1 + + # Ensure any old restore user is removed first, if it exists. + # If it doesn't exist it may return error, so do not exit the + # script if that's the case. + delete_restore_user "dont_exit_on_error" + + $MYSQL --execute="GRANT SELECT ON *.* TO ${RESTORE_USER}@'%' IDENTIFIED BY '${RESTORE_PW}';" 2>>$RESTORE_LOG + if [[ "$?" -eq 0 ]] + then + $MYSQL --execute="GRANT ALL ON ${restore_db}.* TO ${RESTORE_USER}@'%' IDENTIFIED BY '${RESTORE_PW}';" 2>>$RESTORE_LOG + if [[ "$?" -ne 0 ]] + then + cat $RESTORE_LOG + echo "Failed to grant restore user ALL permissions on database ${restore_db}" + return 1 + fi + else + cat $RESTORE_LOG + echo "Failed to grant restore user select permissions on all databases" + return 1 + fi +} + +# Delete temporary restore user +delete_restore_user() { + error_handling=$1 + + $MYSQL --execute="DROP USER ${RESTORE_USER}@'%';" 2>>$RESTORE_LOG + if [[ "$?" -ne 0 ]] + then + if [ "$error_handling" == "exit_on_error" ] + then + cat $RESTORE_LOG + echo "Failed to delete temporary restore user - needs attention to avoid a security hole" + return 1 + fi + fi +} + +#Restore a single database +restore_single_db() { + SINGLE_DB_NAME=$1 + TMP_DIR=$2 + + if [[ -z "$SINGLE_DB_NAME" ]] + then + echo "Restore single DB called but with wrong parameter." + return 1 + fi + + SQL_FILE=$(_get_sql_file $TMP_DIR) + if [ ! -z $SQL_FILE ]; then + # Restoring a single database requires us to create a temporary user + # which has capability to only restore that ONE database. One gotcha + # is that the mysql command to restore the database is going to throw + # errors because of all the other databases that it cannot access. So + # because of this reason, the --force option is used to prevent the + # command from stopping on an error. + create_restore_user $SINGLE_DB_NAME + if [[ $? -ne 0 ]] + then + echo "Restore $SINGLE_DB_NAME failed create restore user." + return 1 + fi + $RESTORE_CMD --force < $SQL_FILE 2>>$RESTORE_LOG + if [[ "$?" -eq 0 ]] + then + echo "Database $SINGLE_DB_NAME Restore successful." + else + cat $RESTORE_LOG + delete_restore_user "exit_on_error" + echo "Database $SINGLE_DB_NAME Restore failed." + return 1 + fi + delete_restore_user "exit_on_error" + if [[ $? -ne 0 ]] + then + echo "Restore $SINGLE_DB_NAME failed delete restore user." + return 1 + fi + if [ -f ${TMP_DIR}/${SINGLE_DB_NAME}_grant.sql ] + then + $MYSQL < ${TMP_DIR}/${SINGLE_DB_NAME}_grant.sql 2>>$RESTORE_LOG + if [[ "$?" -eq 0 ]] + then + if ! $MYSQL --execute="FLUSH PRIVILEGES;"; then + echo "Failed to flush privileges for $SINGLE_DB_NAME." + return 1 + fi + echo "Database $SINGLE_DB_NAME Permission Restore successful." + else + cat $RESTORE_LOG + echo "Database $SINGLE_DB_NAME Permission Restore failed." + return 1 + fi + else + echo "There is no permission file available for $SINGLE_DB_NAME" + return 1 + fi + else + echo "There is no database file available to restore from" + return 1 + fi + return 0 +} + +#Restore all the databases +restore_all_dbs() { + TMP_DIR=$1 + + SQL_FILE=$(_get_sql_file $TMP_DIR) + if [ ! -z $SQL_FILE ]; then + # Check the scope of the archive. + SCOPE=$(echo ${SQL_FILE} | awk -F'.' '{print $(NF-1)}') + if [[ "${SCOPE}" != "all" ]]; then + # This is just a single database backup. The user should + # instead use the single database restore option. + echo "Cannot use the restore all option for an archive containing only a single database." + echo "Please use the single database restore option." + return 1 + fi + + $MYSQL < $SQL_FILE 2>$RESTORE_LOG + if [[ "$?" -eq 0 ]] + then + echo "Databases $( echo $DBS | tr -d '\n') Restore successful." + else + cat $RESTORE_LOG + echo "Databases $( echo $DBS | tr -d '\n') Restore failed." + return 1 + fi + if [[ -f ${TMP_DIR}/grants.sql ]] + then + $MYSQL < ${TMP_DIR}/grants.sql 2>$RESTORE_LOG + if [[ "$?" -eq 0 ]] + then + if ! $MYSQL --execute="FLUSH PRIVILEGES;"; then + echo "Failed to flush privileges." + return 1 + fi + echo "Databases Permission Restore successful." + else + cat $RESTORE_LOG + echo "Databases Permission Restore failed." + return 1 + fi + else + echo "There is no permission file available" + return 1 + fi + else + echo "There is no database file available to restore from" + return 1 + fi + return 0 +} + +# Call the CLI interpreter, providing the archive directory path and the +# user arguments passed in +cli_main ${ARGS[@]} diff --git a/mariadb/templates/bin/_start.py.tpl b/mariadb/templates/bin/_start.py.tpl new file mode 100644 index 0000000000..90fea03f9e --- /dev/null +++ b/mariadb/templates/bin/_start.py.tpl @@ -0,0 +1,1025 @@ +#!/usr/bin/python3 + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +import errno +import logging +import os +import secrets +import select +import signal +import subprocess # nosec +import socket +import sys +import tempfile +import time +import threading +from datetime import datetime, timedelta + +import configparser +import iso8601 +import kubernetes.client +import kubernetes.config + +# Create logger, console handler and formatter +logger = logging.getLogger('OpenStack-Helm Mariadb') +logger.setLevel(logging.INFO) +ch = logging.StreamHandler() +ch.setLevel(logging.INFO) +formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s') + +# Set the formatter and add the handler +ch.setFormatter(formatter) +logger.addHandler(ch) + +# Get the local hostname +local_hostname = socket.gethostname() +logger.info("This instance hostname: {0}".format(local_hostname)) + +# Get local node IP address +local_ip = socket.gethostbyname(local_hostname) +logger.info("This instance IP address: {0}".format(local_ip)) + +# Get the instance number +instance_number = local_hostname.split("-")[-1] +logger.info("This instance number: {0}".format(instance_number)) + +# Setup k8s client credentials and check api version +kubernetes.config.load_incluster_config() +kubernetes_version = kubernetes.client.VersionApi().get_code().git_version +logger.info("Kubernetes API Version: {0}".format(kubernetes_version)) +k8s_api_instance = kubernetes.client.CoreV1Api() + +# Setup secrets generator +secretsGen = secrets.SystemRandom() + +def check_env_var(env_var): + """Check if an env var exists. + + Keyword arguments: + env_var -- the env var to check for the existance of + """ + if env_var in os.environ: + return True + else: + logger.critical("environment variable \"{0}\" not set".format(env_var)) + sys.exit(1) + + +# Set some variables from env vars injected into the container +if check_env_var("STATE_CONFIGMAP"): + state_configmap_name = os.environ['STATE_CONFIGMAP'] + logger.info("Will use \"{0}\" configmap for cluster state info".format( + state_configmap_name)) +if check_env_var("POD_NAMESPACE"): + pod_namespace = os.environ['POD_NAMESPACE'] +if check_env_var("DIRECT_SVC_NAME"): + direct_svc_name = os.environ['DIRECT_SVC_NAME'] +if check_env_var("MARIADB_REPLICAS"): + mariadb_replicas = os.environ['MARIADB_REPLICAS'] +if check_env_var("POD_NAME_PREFIX"): + pod_name_prefix = os.environ['POD_NAME_PREFIX'] +if check_env_var("DISCOVERY_DOMAIN"): + discovery_domain = os.environ['DISCOVERY_DOMAIN'] +if check_env_var("WSREP_PORT"): + wsrep_port = os.environ['WSREP_PORT'] +if check_env_var("MYSQL_DBADMIN_USERNAME"): + mysql_dbadmin_username = os.environ['MYSQL_DBADMIN_USERNAME'] +if check_env_var("MYSQL_DBADMIN_PASSWORD"): + mysql_dbadmin_password = os.environ['MYSQL_DBADMIN_PASSWORD'] +if check_env_var("MYSQL_DBSST_USERNAME"): + mysql_dbsst_username = os.environ['MYSQL_DBSST_USERNAME'] +if check_env_var("MYSQL_DBSST_PASSWORD"): + mysql_dbsst_password = os.environ['MYSQL_DBSST_PASSWORD'] +if check_env_var("MYSQL_DBAUDIT_USERNAME"): + mysql_dbaudit_username = os.environ['MYSQL_DBAUDIT_USERNAME'] +else: + mysql_dbaudit_username = '' +if check_env_var("MYSQL_DBAUDIT_PASSWORD"): + mysql_dbaudit_password = os.environ['MYSQL_DBAUDIT_PASSWORD'] + +mysql_x509 = os.getenv('MARIADB_X509', "") +MYSQL_SSL_CMD_OPTS=["--ssl-verify-server-cert=false", + "--ssl-ca=/etc/mysql/certs/ca.crt", + "--ssl-key=/etc/mysql/certs/tls.key", + "--ssl-cert=/etc/mysql/certs/tls.crt"] + +if mysql_dbadmin_username == mysql_dbsst_username: + logger.critical( + "The dbadmin username should not match the sst user username") + sys.exit(1) + +# Set some variables for tuneables +cluster_leader_ttl = int(os.environ['CLUSTER_LEADER_TTL']) +state_configmap_update_period = 10 +default_sleep = 20 + +# set one name for all commands, avoid "magic names" +MYSQL_BINARY_NAME='mysqld' + + +def ensure_state_configmap(pod_namespace, configmap_name, configmap_body): + """Ensure the state configmap exists. + + Keyword arguments: + pod_namespace -- the namespace to house the configmap + configmap_name -- the configmap name + configmap_body -- the configmap body + """ + try: + k8s_api_instance.read_namespaced_config_map( + name=configmap_name, namespace=pod_namespace) + return True + except: + k8s_api_instance.create_namespaced_config_map( + namespace=pod_namespace, body=configmap_body) + + return False + + +def run_cmd_with_logging(popenargs, + logger, + stdout_log_level=logging.INFO, + stderr_log_level=logging.INFO, + **kwargs): + """Run subprocesses and stream output to logger.""" + child = subprocess.Popen( # nosec + popenargs, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **kwargs) + log_level = { + child.stdout: stdout_log_level, + child.stderr: stderr_log_level + } + + def check_io(): + ready_to_read = select.select([child.stdout, child.stderr], [], [], + 1000)[0] + for io in ready_to_read: + line = io.readline().decode() + logger.log(log_level[io], line[:-1]) + + while child.poll( + ) is None: # keep checking stdout/stderr until the child exits + check_io() + check_io() # check again to catch anything after the process exits + return child.wait() + + +def wait_mysql_status(delay=30): + logger.info("Start checking mariadb status") + i = 0 + res = 1 + while True: + logger.info("Checking mysql status {0}".format(i)) + cmd = ['mysql', + "--defaults-file=/etc/mysql/admin_user.cnf", + "--host=localhost"] + if mysql_x509: + cmd.extend(MYSQL_SSL_CMD_OPTS) + cmd.extend(["--execute=status"]) + res = run_cmd_with_logging(cmd, logger) + if res == 0: + logger.info("mariadb status check passed") + break + else: + logger.info("mariadb status check failed") + i += 1 + time.sleep(delay) + + +def stop_mysqld(): + """Stop mysqld, assuming pid file in default location.""" + logger.info("Shutting down any mysqld instance if required") + mysqld_pidfile_path = "/var/lib/mysql/{0}.pid".format(local_hostname) + + def is_pid_running(pid): + if os.path.isdir('/proc/{0}'.format(pid)): + return True + return False + + def is_pid_mysqld(pid): + with open('/proc/{0}/comm'.format(pid), "r") as mysqld_pidfile: + comm = mysqld_pidfile.readlines()[0].rstrip('\n') + if comm.startswith(MYSQL_BINARY_NAME): + return True + else: + return False + + if not os.path.isfile(mysqld_pidfile_path): + logger.debug("No previous pid file found for mysqld") + return + + if os.stat(mysqld_pidfile_path).st_size == 0: + logger.info( + "{0} file is empty, removing it".format(mysqld_pidfile_path)) + os.remove(mysqld_pidfile_path) + return + + logger.info( + "Previous pid file found for mysqld, attempting to shut it down") + + with open(mysqld_pidfile_path, "r") as mysqld_pidfile: + mysqld_pid = int(mysqld_pidfile.readlines()[0].rstrip('\n')) + + if not is_pid_running(mysqld_pid): + logger.info( + "Mysqld was not running with pid {0}, going to remove stale " + "file".format(mysqld_pid)) + os.remove(mysqld_pidfile_path) + return + if not is_pid_mysqld(mysqld_pid): + logger.error( + "pidfile process is not mysqld, removing pidfile and panic") + os.remove(mysqld_pidfile_path) + sys.exit(1) + + logger.info("pid from pidfile is mysqld") + os.kill(mysqld_pid, 15) + try: + pid, status = os.waitpid(mysqld_pid, 0) + except OSError as err: + # The process has already exited + if err.errno == errno.ECHILD: + return + else: + raise + logger.info("Mysqld stopped: pid = {0}, " + "exit status = {1}".format(pid, status)) + + +def mysqld_write_cluster_conf(mode='run'): + """Write out dynamic cluster config. + + Keyword arguments: + mode -- whether we are writing the cluster config for the cluster to 'run' + or 'bootstrap' (default 'run') + """ + logger.info("Setting up cluster config") + cluster_config = configparser.ConfigParser() + cluster_config['mysqld'] = {} + cluster_config_params = cluster_config['mysqld'] + wsrep_cluster_members = [] + for node in range(int(mariadb_replicas)): + node_hostname = "{0}-{1}".format(pod_name_prefix, node) + if local_hostname == node_hostname: + cluster_config_params['wsrep_node_address'] = local_ip + wsrep_node_name = "{0}.{1}".format(node_hostname, discovery_domain) + cluster_config_params['wsrep_node_name'] = wsrep_node_name + + if mode == 'run': + cluster_config_params['wsrep_cluster_address'] = "gcomm://{0}:{1}".format( + discovery_domain, wsrep_port) + + else: + cluster_config_params['wsrep_cluster_address'] = "gcomm://" + cluster_config_file = '/etc/mysql/conf.d/10-cluster-config.cnf' + logger.info( + "Writing out cluster config to: {0}".format(cluster_config_file)) + with open(cluster_config_file, 'w') as configfile: + cluster_config.write(configfile) + + +# Function to setup mysqld +def mysqld_bootstrap(): + """Bootstrap the db if no data found in the 'bootstrap_test_dir'""" + logger.info("Boostrapping Mariadb") + mysql_data_dir = '/var/lib/mysql' + bootstrap_test_dir = "{0}/mysql".format(mysql_data_dir) + if not os.path.isdir(bootstrap_test_dir): + stop_mysqld() + mysqld_write_cluster_conf(mode='bootstrap') + run_cmd_with_logging([ + 'mysql_install_db', '--user=mysql', + "--datadir={0}".format(mysql_data_dir) + ], logger) + if not mysql_dbaudit_username: + template = ( + # NOTE: since mariadb 10.4.13 definer of view + # mysql.user is not root but mariadb.sys user + # it is safe not to remove it because the account by default + # is locked and cannot login + "DELETE FROM mysql.user WHERE user != 'mariadb.sys' ;\n" # nosec + "CREATE OR REPLACE USER '{0}'@'%' IDENTIFIED BY \'{1}\' ;\n" + "GRANT ALL ON *.* TO '{0}'@'%' {4} WITH GRANT OPTION; \n" + "DROP DATABASE IF EXISTS test ;\n" + "CREATE OR REPLACE USER '{2}'@'127.0.0.1' IDENTIFIED BY '{3}';\n" + "GRANT PROCESS, RELOAD, LOCK TABLES, REPLICATION CLIENT ON *.* TO '{2}'@'127.0.0.1';\n" + "FLUSH PRIVILEGES ;\n" + "SHUTDOWN ;".format(mysql_dbadmin_username, mysql_dbadmin_password, + mysql_dbsst_username, mysql_dbsst_password, + mysql_x509)) + else: + template = ( + "DELETE FROM mysql.user WHERE user != 'mariadb.sys' ;\n" # nosec + "CREATE OR REPLACE USER '{0}'@'%' IDENTIFIED BY \'{1}\' ;\n" + "GRANT ALL ON *.* TO '{0}'@'%' {6} WITH GRANT OPTION;\n" + "DROP DATABASE IF EXISTS test ;\n" + "CREATE OR REPLACE USER '{2}'@'127.0.0.1' IDENTIFIED BY '{3}';\n" + "GRANT PROCESS, RELOAD, LOCK TABLES, REPLICATION CLIENT ON *.* TO '{2}'@'127.0.0.1' ;\n" + "CREATE OR REPLACE USER '{4}'@'%' IDENTIFIED BY '{5}';\n" + "GRANT SELECT ON *.* TO '{4}'@'%' {6};\n" + "FLUSH PRIVILEGES ;\n" + "SHUTDOWN ;".format(mysql_dbadmin_username, mysql_dbadmin_password, + mysql_dbsst_username, mysql_dbsst_password, + mysql_dbaudit_username, mysql_dbaudit_password, + mysql_x509)) + bootstrap_sql_file = tempfile.NamedTemporaryFile(suffix='.sql').name + with open(bootstrap_sql_file, 'w') as f: + f.write(template) + f.close() + run_cmd_with_logging([ + MYSQL_BINARY_NAME, '--user=mysql', '--bind-address=127.0.0.1', + '--wsrep_cluster_address=gcomm://', + "--init-file={0}".format(bootstrap_sql_file) + ], logger) + os.remove(bootstrap_sql_file) + else: + logger.info("Skipping bootstrap as {0} directory is present".format( + bootstrap_test_dir)) + + +def safe_update_configmap(configmap_dict, configmap_patch): + """Update a configmap with locking. + + Keyword arguments: + configmap_dict -- a dict representing the configmap to be patched + configmap_patch -- a dict containign the patch + """ + logger.debug("Safe Patching configmap") + # NOTE(portdirect): Explictly set the resource version we are patching to + # ensure nothing else has modified the confimap since we read it. + configmap_patch['metadata']['resourceVersion'] = configmap_dict[ + 'metadata']['resource_version'] + try: + api_response = k8s_api_instance.patch_namespaced_config_map( + name=state_configmap_name, + namespace=pod_namespace, + body=configmap_patch) + return True + except kubernetes.client.rest.ApiException as error: + if error.status == 409: + # This status code indicates a collision trying to write to the + # config map while another instance is also trying the same. + logger.warning("Collision writing configmap: {0}".format(error)) + # This often happens when the replicas were started at the same + # time, and tends to be persistent. Sleep with some random + # jitter value briefly to break the synchronization. + naptime = secretsGen.uniform(0.8,1.2) + time.sleep(naptime) + else: + logger.error("Failed to set configmap: {0}".format(error)) + return error + + +def set_configmap_annotation(key, value): + """Update a configmap's annotations via patching. + + Keyword arguments: + key -- the key to be patched + value -- the value to give the key + """ + logger.debug("Setting configmap annotation key={0} value={1}".format( + key, value)) + configmap_dict = k8s_api_instance.read_namespaced_config_map( + name=state_configmap_name, namespace=pod_namespace).to_dict() + configmap_patch = {'metadata': {'annotations': {}}} + configmap_patch['metadata']['annotations'][key] = value + return safe_update_configmap( + configmap_dict=configmap_dict, configmap_patch=configmap_patch) + + +def set_configmap_data(key, value): + """Update a configmap's data via patching. + + Keyword arguments: + key -- the key to be patched + value -- the value to give the key + """ + logger.debug("Setting configmap data key={0} value={1}".format(key, value)) + configmap_dict = k8s_api_instance.read_namespaced_config_map( + name=state_configmap_name, namespace=pod_namespace).to_dict() + configmap_patch = {'data': {}, 'metadata': {}} + configmap_patch['data'][key] = value + return safe_update_configmap( + configmap_dict=configmap_dict, configmap_patch=configmap_patch) + + +def get_configmap_value(key, type='data'): + """Get a configmap's key's value. + + Keyword arguments: + key -- the key to retrive the data from + type -- the type of data to retrive from the configmap, can either be 'data' + or an 'annotation'. (default data) + """ + state_configmap = k8s_api_instance.read_namespaced_config_map( + name=state_configmap_name, namespace=pod_namespace) + state_configmap_dict = state_configmap.to_dict() + if type == 'data': + state_configmap_data = state_configmap_dict['data'] + elif type == 'annotation': + state_configmap_data = state_configmap_dict['metadata']['annotations'] + else: + logger.error( + "Unknown data type \"{0}\" reqested for retrival".format(type)) + return False + if state_configmap_data and key in state_configmap_data: + return state_configmap_data[key] + else: + return None + + +def get_cluster_state(): + """Get the current cluster state from a configmap, creating the configmap + if it does not already exist. + """ + logger.info("Getting cluster state") + state = None + while state is None: + try: + state = get_configmap_value( + type='annotation', + key='openstackhelm.openstack.org/cluster.state') + logger.info( + "The cluster is currently in \"{0}\" state.".format(state)) + except: + logger.info("The cluster configmap \"{0}\" does not exist.".format( + state_configmap_name)) + time.sleep(default_sleep) + leader_expiry_raw = datetime.utcnow() + timedelta( + seconds=cluster_leader_ttl) + leader_expiry = "{0}Z".format(leader_expiry_raw.isoformat("T")) + if check_for_active_nodes(): + # NOTE(portdirect): here we make the assumption that the 1st pod + # in an existing statefulset is the one to adopt as leader. + leader = "{0}-0".format("-".join( + local_hostname.split("-")[:-1])) + state = "live" + logger.info( + "The cluster is running already though unmanaged \"{0}\" will be declared leader in a \"{1}\" state". + format(leader, state)) + else: + leader = local_hostname + state = "new" + logger.info( + "The cluster is new \"{0}\" will be declared leader in a \"{1}\" state". + format(leader, state)) + + initial_configmap_body = { + "apiVersion": "v1", + "kind": "ConfigMap", + "metadata": { + "name": state_configmap_name, + "annotations": { + "openstackhelm.openstack.org/cluster.state": state, + "openstackhelm.openstack.org/leader.node": leader, + "openstackhelm.openstack.org/leader.expiry": + leader_expiry, + "openstackhelm.openstack.org/reboot.node": "" + } + }, + "data": {} + } + ensure_state_configmap( + pod_namespace=pod_namespace, + configmap_name=state_configmap_name, + configmap_body=initial_configmap_body) + return state + + +def declare_myself_cluster_leader(): + """Declare the current pod as the cluster leader.""" + logger.info("Declaring myself current cluster leader") + leader_expiry_raw = datetime.utcnow() + timedelta( + seconds=cluster_leader_ttl) + leader_expiry = "{0}Z".format(leader_expiry_raw.isoformat("T")) + set_configmap_annotation( + key='openstackhelm.openstack.org/leader.node', value=local_hostname) + set_configmap_annotation( + key='openstackhelm.openstack.org/leader.expiry', value=leader_expiry) + + +def deadmans_leader_election(): + """Run a simplisic deadmans leader election.""" + leader_node = get_configmap_value( + type='annotation', key='openstackhelm.openstack.org/leader.node') + leader_expiry = get_configmap_value( + type='annotation', key='openstackhelm.openstack.org/leader.expiry') + if iso8601.parse_date(leader_expiry).replace( + tzinfo=None) < datetime.utcnow().replace(tzinfo=None): + logger.info("Current cluster leader has expired") + declare_myself_cluster_leader() + elif local_hostname == leader_node: + logger.info("Renewing cluster leader lease") + declare_myself_cluster_leader() + + +def get_grastate_val(key): + """Extract data from grastate.dat. + + Keyword arguments: + key -- the key to extract the value of + """ + logger.debug("Reading grastate.dat key={0}".format(key)) + try: + # This attempts to address a potential race condition with the initial + # creation of the grastate.date file where the file would exist + # however, it is not immediately populated. Testing indicated it could + # take 15-20 seconds for the file to be populated. So loop and keep + # checking up to 60 seconds. If it still isn't populated afterwards, + # the IndexError will still occur as we are seeing now without the loop. + time_end = time.time() + 60 + while time.time() < time_end: + with open("/var/lib/mysql/grastate.dat", "r") as myfile: + grastate_raw = [s.strip() for s in myfile.readlines()] + if grastate_raw: + break + time.sleep(1) + return [i for i in grastate_raw + if i.startswith("{0}:".format(key))][0].split(':')[1].strip() + except IndexError: + logger.error( + "IndexError: Unable to find %s with ':' in grastate.dat", key) + raise + + +def set_grastate_val(key, value): + """Set values in grastate.dat. + + Keyword arguments: + key -- the key to set the value of + value -- the value to set the key to + """ + logger.debug("Updating grastate.dat key={0} value={1}".format(key, value)) + with open("/var/lib/mysql/grastate.dat", "r") as sources: + lines = sources.readlines() + for line_num, line_content in enumerate(lines): + if line_content.startswith("{0}:".format(key)): + line_content = "{0}: {1}\n".format(key, value) + lines[line_num] = line_content + with open("/var/lib/mysql/grastate.dat", "w") as sources: + for line in lines: + sources.write(line) + + +def update_grastate_configmap(): + """Update state configmap with grastate.dat info.""" + while not os.path.exists('/var/lib/mysql/grastate.dat'): + time.sleep(1) + logger.info("Updating grastate configmap") + grastate = dict() + grastate['version'] = get_grastate_val(key='version') + grastate['uuid'] = get_grastate_val(key='uuid') + grastate['seqno'] = get_grastate_val(key='seqno') + grastate['safe_to_bootstrap'] = get_grastate_val(key='safe_to_bootstrap') + grastate['sample_time'] = "{0}Z".format(datetime.utcnow().isoformat("T")) + for grastate_key, grastate_value in list(grastate.items()): + configmap_key = "{0}.{1}".format(grastate_key, local_hostname) + if get_configmap_value(type='data', key=configmap_key) != grastate_value: + set_configmap_data(key=configmap_key, value=grastate_value) + + +def update_grastate_on_restart(): + """Update the grastate.dat on node restart.""" + logger.info("Updating grastate info for node") + if os.path.exists('/var/lib/mysql/grastate.dat'): + if get_grastate_val(key='seqno') == '-1': + logger.info( + "Node shutdown was not clean, getting position via wsrep-recover" + ) + + def recover_wsrep_position(): + """Extract recovered wsrep position from uncleanly exited node.""" + wsrep_recover = subprocess.Popen( # nosec + [ + MYSQL_BINARY_NAME, '--bind-address=127.0.0.1', + '--wsrep_cluster_address=gcomm://', '--wsrep-recover' + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding="utf-8") + out, err = wsrep_recover.communicate() + wsrep_rec_pos = None + # NOTE: communicate() returns a tuple (stdout_data, stderr_data). + # The data will be strings if streams were opened in text mode; + # otherwise, bytes. If it is bytes, we should decode and get a + # str for the err.split() to not error below. + if isinstance(err, bytes): + err = err.decode('utf-8') + for item in err.split("\n"): + logger.info("Recovering wsrep position: {0}".format(item)) + if "WSREP: Recovered position:" in item: + line = item.strip().split() + wsrep_rec_pos = line[-1].split(':')[-1] + if wsrep_rec_pos is None: + logger.error("WSREP_REC_POS position could not be found.") + raise Exception("WSREP_REC_POS position could not be found.") + return wsrep_rec_pos + + set_grastate_val(key='seqno', value=recover_wsrep_position()) + else: + logger.info("Node shutdown was clean, using grastate.dat") + + update_grastate_configmap() + + else: + logger.info("No grastate.dat exists I am a new node") + + +def get_active_endpoints(endpoints_name=direct_svc_name, + namespace=pod_namespace): + """Returns a list of active endpoints. + + Keyword arguments: + endpoints_name -- endpoints to check for active backends + (default direct_svc_name) + namespace -- namespace to check for endpoints (default pod_namespace) + """ + try: + endpoints = k8s_api_instance.read_namespaced_endpoints( + name=endpoints_name, namespace=pod_namespace) + except kubernetes.client.rest.ApiException as error: + logger.error("Failed to get mariadb service with error: {0}".format(error)) + raise error + endpoints_dict = endpoints.to_dict() + active_endpoints = [] + if endpoints_dict['subsets']: + active_endpoints = [s['addresses'] for s in endpoints_dict['subsets'] if 'addresses' in s + ][0] + return active_endpoints + + +def check_for_active_nodes(endpoints_name=direct_svc_name, + namespace=pod_namespace): + """Check K8s endpoints to see if there are active Mariadb Instances. + + Keyword arguments: + endpoints_name -- endpoints to check for active backends + (default direct_svc_name) + namespace -- namespace to check for endpoints (default pod_namespace) + """ + logger.info("Checking for active nodes") + active_endpoints = get_active_endpoints() + if active_endpoints and len(active_endpoints) >= 1: + logger.info("Amount of active endpoints: {0}".format(len(active_endpoints))) + return True + else: + logger.info("Amount of active endpoints: 0") + return False + + +def check_if_cluster_data_is_fresh(): + """Check if the state_configmap is both current and reasonably stable.""" + logger.info("Checking to see if cluster data is fresh") + state_configmap = k8s_api_instance.read_namespaced_config_map( + name=state_configmap_name, namespace=pod_namespace) + state_configmap_dict = state_configmap.to_dict() + sample_times = dict() + for key, value in list(state_configmap_dict['data'].items()): + keyitems = key.split('.') + key = keyitems[0] + node = keyitems[1] + if key == 'sample_time': + sample_times[node] = value + sample_time_ok = True + for key, value in list(sample_times.items()): + sample_time = iso8601.parse_date(value).replace(tzinfo=None) + # NOTE(vsaienko): give some time on resolving configmap update conflicts + sample_cutoff_time = datetime.utcnow().replace( + tzinfo=None) - timedelta(seconds=5*state_configmap_update_period) + if not sample_time >= sample_cutoff_time: + logger.info( + "The data we have from the cluster is too old to make a " + "decision for node {0}".format(key)) + sample_time_ok = False + else: + logger.info( + "The data we have from the cluster is ok for node {0}".format( + key)) + return sample_time_ok + + +def get_nodes_with_highest_seqno(): + """Find out which node(s) has the highest sequence number and return + them in an array.""" + logger.info("Getting the node(s) with highest seqno from configmap.") + # We can proceed only when we get seqno from all nodes, and if seqno is + # -1 it means we didn't get it correctly, the shutdown was not clean and we need + # to wait for a value taken by wsrep recover. + while True: + state_configmap = k8s_api_instance.read_namespaced_config_map( + name=state_configmap_name, namespace=pod_namespace) + state_configmap_dict = state_configmap.to_dict() + seqnos = dict() + for key, value in list(state_configmap_dict['data'].items()): + keyitems = key.split('.') + key = keyitems[0] + node = keyitems[1] + if key == 'seqno': + #Explicit casting to integer to have resulting list of integers for correct comparison + seqnos[node] = int(value) + max_seqno = max(seqnos.values()) + max_seqno_nodes = sorted([k for k, v in list(seqnos.items()) if v == max_seqno]) + if [x for x in seqnos.values() if x < 0 ]: + logger.info("Thq seqno for some nodes is < 0, can't make a decision about leader. Node seqnums: %s", seqnos) + time.sleep(state_configmap_update_period) + continue + return max_seqno_nodes + + +def resolve_leader_node(nodename_array): + """From the given nodename array, determine which node is the leader + by choosing the node which has a hostname with the lowest number at + the end of it. If by chance there are two nodes with the same number + then the first one encountered will be chosen.""" + logger.info("Returning the node with the lowest hostname") + lowest = sys.maxsize + leader = nodename_array[0] + for nodename in nodename_array: + nodenum = int(nodename[nodename.rindex('-') + 1:]) + logger.info("Nodename %s has nodenum %d", nodename, nodenum) + if nodenum < lowest: + lowest = nodenum + leader = nodename + logger.info("Resolved leader is %s", leader) + return leader + + +def check_if_i_lead(): + """Check on full restart of cluster if this node should lead the cluster + reformation.""" + logger.info("Checking to see if I lead the cluster for reboot") + # as we sample on the update period - we sample for a full cluster + # leader election period as a simplistic way of ensureing nodes are + # reliably checking in following full restart of cluster. + count = cluster_leader_ttl / state_configmap_update_period + counter = 0 + while counter < count: + if check_if_cluster_data_is_fresh(): + counter += 1 + else: + counter = 0 + time.sleep(state_configmap_update_period) + logger.info( + "Cluster info has been uptodate {0} times out of the required " + "{1}".format(counter, count)) + max_seqno_nodes = get_nodes_with_highest_seqno() + leader_node = resolve_leader_node(max_seqno_nodes) + if (local_hostname == leader_node and not check_for_active_nodes() + and get_cluster_state() == 'live'): + logger.info("I lead the cluster. Setting cluster state to reboot.") + set_configmap_annotation( + key='openstackhelm.openstack.org/cluster.state', value='reboot') + set_configmap_annotation( + key='openstackhelm.openstack.org/reboot.node', value=local_hostname) + return True + elif local_hostname == leader_node: + logger.info("The cluster is already rebooting") + return False + else: + logger.info("{0} leads the cluster".format(leader_node)) + return False + + +def monitor_cluster(stop_event): + """Function to kick off grastate configmap updating thread""" + while True: + if stop_event.is_set(): + logger.info("Stopped monitor_cluster thread") + break + try: + update_grastate_configmap() + except Exception as error: + logger.error("Error updating grastate configmap: {0}".format(error)) + time.sleep(state_configmap_update_period) + +# Stop event +stop_event = threading.Event() + +# Setup the thread for the cluster monitor +monitor_cluster_thread = threading.Thread(target=monitor_cluster, args=(stop_event,)) +monitor_cluster_thread.daemon = True + + +def launch_cluster_monitor(): + """Launch grastate configmap updating thread""" + if not monitor_cluster_thread.is_alive(): + monitor_cluster_thread.start() + + +def leader_election(stop_event): + """Function to kick off leader election thread""" + while True: + if stop_event.is_set(): + logger.info("Stopped leader_election thread") + break + try: + deadmans_leader_election() + except Exception as error: + logger.error("Error electing leader: {0}".format(error)) + time.sleep(cluster_leader_ttl / 2) + + +# Setup the thread for the leader election +leader_election_thread = threading.Thread(target=leader_election, args=(stop_event,)) +leader_election_thread.daemon = True + + +def launch_leader_election(): + """Launch leader election thread""" + if not leader_election_thread.is_alive(): + leader_election_thread.start() + + +def run_mysqld(cluster='existing'): + """Launch the mysqld instance for the pod. This will also run mysql upgrade + if we are the 1st replica, and the rest of the cluster is already running. + This senario will be triggerd either following a rolling update, as this + works in reverse order for statefulset. Or restart of the 1st instance, in + which case the comand should be a no-op. + + Keyword arguments: + cluster -- whether we going to form a cluster 'new' or joining an existing + cluster 'existing' (default 'existing') + """ + stop_mysqld() + mysqld_write_cluster_conf(mode='run') + launch_leader_election() + launch_cluster_monitor() + mysqld_cmd = [MYSQL_BINARY_NAME, '--user=mysql'] + if cluster == 'new': + mysqld_cmd.append('--wsrep-new-cluster') + + mysql_data_dir = '/var/lib/mysql' + db_test_dir = "{0}/mysql".format(mysql_data_dir) + if os.path.isdir(db_test_dir): + logger.info("Setting the admin passwords to the current value and upgrade mysql if needed") + if not mysql_dbaudit_username: + template = ( + "CREATE OR REPLACE USER '{0}'@'%' IDENTIFIED BY \'{1}\' ;\n" + "GRANT ALL ON *.* TO '{0}'@'%' {4} WITH GRANT OPTION ;\n" + "CREATE OR REPLACE USER '{2}'@'127.0.0.1' IDENTIFIED BY '{3}' ;\n" + "GRANT PROCESS, RELOAD, LOCK TABLES, REPLICATION CLIENT ON *.* TO '{2}'@'127.0.0.1' ;\n" + "FLUSH PRIVILEGES ;".format(mysql_dbadmin_username, mysql_dbadmin_password, + mysql_dbsst_username, mysql_dbsst_password, + mysql_x509)) + else: + template = ( + "CREATE OR REPLACE USER '{0}'@'%' IDENTIFIED BY \'{1}\' ;\n" + "GRANT ALL ON *.* TO '{0}'@'%' {6} WITH GRANT OPTION ;\n" + "CREATE OR REPLACE USER '{2}'@'127.0.0.1' IDENTIFIED BY '{3}' ;\n" + "GRANT PROCESS, RELOAD, LOCK TABLES, REPLICATION CLIENT ON *.* TO '{2}'@'127.0.0.1' ;\n" + "CREATE OR REPLACE USER '{4}'@'%' IDENTIFIED BY '{5}' ;\n" + "GRANT SELECT ON *.* TO '{4}'@'%' {6};\n" + "FLUSH PRIVILEGES ;".format(mysql_dbadmin_username, mysql_dbadmin_password, + mysql_dbsst_username, mysql_dbsst_password, + mysql_dbaudit_username, mysql_dbaudit_password, + mysql_x509)) + bootstrap_sql_file = tempfile.NamedTemporaryFile(suffix='.sql').name + with open(bootstrap_sql_file, 'w') as f: + f.write(template) + f.close() + run_cmd_with_logging_thread = threading.Thread(target=run_cmd_with_logging, args=([ + MYSQL_BINARY_NAME, '--bind-address=127.0.0.1', '--wsrep-on=false', + "--init-file={0}".format(bootstrap_sql_file) + ], logger)) + run_cmd_with_logging_thread.start() + wait_mysql_status() + logger.info("Upgrading local mysql instance") + upgrade_cmd=['mysql_upgrade', '--skip-write-binlog', + "--user={0}".format(mysql_dbadmin_username), + "--password={0}".format(mysql_dbadmin_password)] + if mysql_x509: + upgrade_cmd.extend(MYSQL_SSL_CMD_OPTS) + upgrade_res = run_cmd_with_logging(upgrade_cmd, logger) + if upgrade_res != 0: + raise Exception('Mysql upgrade failed, cannot proceed') + stop_mysqld() + os.remove(bootstrap_sql_file) + else: + logger.info( + "This is a fresh node joining the cluster for the 1st time, not attempting to set admin passwords or upgrading" + ) + + logger.info("Launching MariaDB") + run_cmd_with_logging(mysqld_cmd, logger) + + +def mysqld_reboot(): + """Reboot a mysqld cluster.""" + declare_myself_cluster_leader() + set_grastate_val(key='safe_to_bootstrap', value='1') + run_mysqld(cluster='new') + + +def sigterm_shutdown(x, y): + """Shutdown the instance of mysqld on shutdown signal.""" + logger.info("Got a sigterm from the container runtime, time to go.") + stop_event.set() + stop_mysqld() + monitor_cluster_thread.join() + leader_election_thread.join() + sys.exit(0) + + +# Register the signal to the handler +signal.signal(signal.SIGTERM, sigterm_shutdown) + +# Main logic loop +if get_cluster_state() == 'new': + leader_node = get_configmap_value( + type='annotation', key='openstackhelm.openstack.org/leader.node') + if leader_node == local_hostname: + set_configmap_annotation( + key='openstackhelm.openstack.org/cluster.state', value='init') + declare_myself_cluster_leader() + launch_leader_election() + mysqld_bootstrap() + update_grastate_configmap() + set_configmap_annotation( + key='openstackhelm.openstack.org/cluster.state', value='live') + run_mysqld(cluster='new') + else: + logger.info("Waiting for cluster to start running") + while not get_cluster_state() == 'live': + time.sleep(default_sleep) + while not check_for_active_nodes(): + time.sleep(default_sleep) + launch_leader_election() + run_mysqld() +elif get_cluster_state() == 'init': + logger.info("Waiting for cluster to start running") + while not get_cluster_state() == 'live': + time.sleep(default_sleep) + while not check_for_active_nodes(): + time.sleep(default_sleep) + launch_leader_election() + run_mysqld() +elif get_cluster_state() == 'live': + logger.info("Cluster has been running starting restore/rejoin") + if not int(mariadb_replicas) > 1: + logger.info( + "There is only a single node in this cluster, we are good to go") + update_grastate_on_restart() + mysqld_reboot() + else: + if check_for_active_nodes(): + logger.info( + "There are currently running nodes in the cluster, we can " + "join them") + run_mysqld() + else: + logger.info("This cluster has lost all running nodes, we need to " + "determine the new lead node") + update_grastate_on_restart() + launch_leader_election() + launch_cluster_monitor() + if check_if_i_lead(): + logger.info("I won the ability to reboot the cluster") + mysqld_reboot() + else: + logger.info( + "Waiting for the lead node to come online before joining " + "it") + while not check_for_active_nodes(): + time.sleep(default_sleep) + set_configmap_annotation( + key='openstackhelm.openstack.org/cluster.state', value='live') + run_mysqld() +elif get_cluster_state() == 'reboot': + reboot_node = get_configmap_value( + type='annotation', key='openstackhelm.openstack.org/reboot.node') + if reboot_node == local_hostname: + logger.info( + "Cluster reboot procedure wasn`t finished. Trying again.") + update_grastate_on_restart() + launch_leader_election() + launch_cluster_monitor() + mysqld_reboot() + else: + logger.info( + "Waiting for the lead node to come online before joining " + "it") + update_grastate_on_restart() + launch_leader_election() + launch_cluster_monitor() + while not check_for_active_nodes(): + time.sleep(default_sleep) + set_configmap_annotation( + key='openstackhelm.openstack.org/cluster.state', value='live') + run_mysqld() +else: + logger.critical("Dont understand cluster state, exiting with error status") + sys.exit(1) diff --git a/mariadb/templates/bin/_start_mariadb_verify_server.sh.tpl b/mariadb/templates/bin/_start_mariadb_verify_server.sh.tpl new file mode 100644 index 0000000000..c633946c93 --- /dev/null +++ b/mariadb/templates/bin/_start_mariadb_verify_server.sh.tpl @@ -0,0 +1,29 @@ +#!/bin/bash -ex + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +log () { + msg_default="Need some text to log" + level_default="INFO" + component_default="Mariadb Backup Verifier" + + msg=${1:-$msg_default} + level=${2:-$level_default} + component=${3:-"$component_default"} + + echo "$(date +'%Y-%m-%d %H:%M:%S,%3N') - ${component} - ${level} - ${msg}" +} + +log "Starting Mariadb server for backup verification..." +mysql_install_db --user=nobody --ldata=/var/lib/mysql >/dev/null 2>&1 +MYSQL_ALLOW_EMPTY_PASSWORD=1 mysqld --user=nobody --verbose >/dev/null 2>&1 diff --git a/mariadb/templates/bin/_test.sh.tpl b/mariadb/templates/bin/_test.sh.tpl new file mode 100644 index 0000000000..536a4213e5 --- /dev/null +++ b/mariadb/templates/bin/_test.sh.tpl @@ -0,0 +1,27 @@ +#!/bin/bash +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +rm -f /tmp/test-success + +mysqlslap \ + --defaults-file=/etc/mysql/test-params.cnf \ + {{ include "helm-toolkit.utils.joinListWithSpace" $.Values.conf.tests.params }} -vv \ + --post-system="touch /tmp/test-success" + +if ! [ -f /tmp/test-success ]; then + exit 1 +fi diff --git a/mariadb/templates/certificates.yaml b/mariadb/templates/certificates.yaml new file mode 100644 index 0000000000..200f974acf --- /dev/null +++ b/mariadb/templates/certificates.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.certificates -}} +{{ dict "envAll" . "service" "oslo_db" "type" "default" | include "helm-toolkit.manifests.certificates" }} +{{- end -}} diff --git a/mariadb/templates/configmap-bin.yaml b/mariadb/templates/configmap-bin.yaml new file mode 100644 index 0000000000..3e80c05ccf --- /dev/null +++ b/mariadb/templates/configmap-bin.yaml @@ -0,0 +1,58 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_bin }} +{{- $envAll := . }} +{{ if eq .Values.endpoints.oslo_db.auth.admin.username .Values.endpoints.oslo_db.auth.sst.username }} +{{ fail "the DB admin username should not match the sst user username" }} +{{ end }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: mariadb-bin +data: +{{- if .Values.images.local_registry.active }} + image-repo-sync.sh: | +{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} +{{- end }} + health.sh: | +{{ tuple "bin/_health.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + start.py: | +{{ tuple "bin/_start.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + test.sh: | +{{ tuple "bin/_test.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- if .Values.conf.backup.enabled }} + backup_mariadb.sh: | +{{ tuple "bin/_backup_mariadb.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + start_verification_server.sh: | +{{ tuple "bin/_start_mariadb_verify_server.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + restore_mariadb.sh: | +{{ tuple "bin/_restore_mariadb.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + backup_main.sh: | +{{ include "helm-toolkit.scripts.db-backup-restore.backup_main" . | indent 4 }} + restore_main.sh: | +{{ include "helm-toolkit.scripts.db-backup-restore.restore_main" . | indent 4 }} +{{- end }} +{{- if .Values.manifests.job_ks_user }} + ks-user.sh: | +{{ include "helm-toolkit.scripts.keystone_user" . | indent 4 }} +{{- end }} +{{- if .Values.manifests.deployment_controller }} + mariadb_controller.py: | +{{ tuple "bin/_mariadb_controller.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} + mariadb-wait-for-cluster.py: | +{{ tuple "bin/_mariadb-wait-for-cluster.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} diff --git a/mariadb/templates/configmap-etc.yaml b/mariadb/templates/configmap-etc.yaml new file mode 100644 index 0000000000..5367f18d9b --- /dev/null +++ b/mariadb/templates/configmap-etc.yaml @@ -0,0 +1,29 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License" ); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_etc }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: mariadb-etc +data: +{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" ( index $envAll.Values.conf.database "my" ) "key" "my.cnf" ) | indent 2 }} +{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" ( index $envAll.Values.conf.database "00_base" ) "key" "00-base.cnf" ) | indent 2 }} +{{- if $envAll.Values.conf.database.config_override }} +{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" ( index $envAll.Values.conf.database "config_override" ) "key" "20-override.cnf" ) | indent 2 }} +{{- end }} +{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" ( index $envAll.Values.conf.database "99_force" ) "key" "99-force.cnf" ) | indent 2 }} +{{- end }} diff --git a/mariadb/templates/configmap-services-tcp.yaml b/mariadb/templates/configmap-services-tcp.yaml new file mode 100644 index 0000000000..0cd6cb1e8a --- /dev/null +++ b/mariadb/templates/configmap-services-tcp.yaml @@ -0,0 +1,24 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_services_tcp }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: mariadb-services-tcp +data: + {{ tuple "oslo_db" "internal" "mysql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }}: "{{ .Release.Namespace }}/{{ tuple "oslo_db" "direct" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}:{{ tuple "oslo_db" "direct" "mysql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}" +{{- end }} diff --git a/mariadb/templates/cron-job-backup-mariadb.yaml b/mariadb/templates/cron-job-backup-mariadb.yaml new file mode 100644 index 0000000000..cb83812543 --- /dev/null +++ b/mariadb/templates/cron-job-backup-mariadb.yaml @@ -0,0 +1,210 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.cron_job_mariadb_backup }} +{{- $envAll := . }} + +{{- $serviceAccountName := "mariadb-backup" }} +{{ tuple $envAll "mariadb_backup" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: batch/v1 +kind: CronJob +metadata: + name: mariadb-backup + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "mariadb-backup" "backup" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + schedule: {{ .Values.jobs.mariadb_backup.cron | quote }} + successfulJobsHistoryLimit: {{ .Values.jobs.mariadb_backup.history.success }} + failedJobsHistoryLimit: {{ .Values.jobs.mariadb_backup.history.failed }} + concurrencyPolicy: Forbid + jobTemplate: + metadata: + labels: +{{ tuple $envAll "mariadb-backup" "backup" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ dict "envAll" $envAll "podName" "mariadb-backup" "containerNames" (list "init" "backup-perms" "mariadb-backup") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{- if .Values.jobs.mariadb_backup.backoffLimit }} + backoffLimit: {{ .Values.jobs.mariadb_backup.backoffLimit }} +{{- end }} +{{- if .Values.jobs.mariadb_backup.activeDeadlineSeconds }} + activeDeadlineSeconds: {{ .Values.jobs.mariadb_backup.activeDeadlineSeconds }} +{{- end }} + template: + metadata: + labels: +{{ tuple $envAll "mariadb-backup" "backup" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 12 }} + spec: +{{ dict "envAll" $envAll "application" "mariadb_backup" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 10 }} + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + shareProcessNamespace: true +{{- if $envAll.Values.pod.tolerations.mariadb.enabled }} +{{ tuple $envAll "mariadb" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 10 }} +{{- end }} +{{- if $envAll.Values.pod.affinity }} +{{- if $envAll.Values.pod.affinity.mariadb_backup }} + affinity: +{{ index $envAll.Values.pod.affinity "mariadb_backup" | toYaml | indent 12}} +{{- end }} +{{- end }} + nodeSelector: + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} + initContainers: +{{ tuple $envAll "mariadb_backup" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 12 }} + - name: backup-perms +{{ tuple $envAll "mariadb_backup" | include "helm-toolkit.snippets.image" | indent 14 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.mariadb_backup | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }} +{{ dict "envAll" $envAll "application" "mariadb_backup" "container" "backup_perms" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 14 }} + command: + - chown + - -R + - "65534:65534" + - $(MARIADB_BACKUP_BASE_DIR) + env: + - name: MARIADB_BACKUP_BASE_DIR + value: {{ .Values.conf.backup.base_path | quote }} + volumeMounts: + - mountPath: /tmp + name: pod-tmp + - mountPath: {{ .Values.conf.backup.base_path }} + name: mariadb-backup-dir + - name: verify-perms +{{ tuple $envAll "mariadb_backup" | include "helm-toolkit.snippets.image" | indent 14 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.mariadb_backup | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }} +{{ dict "envAll" $envAll "application" "mariadb_backup" "container" "verify_perms" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 14 }} + command: + - chown + - -R + - "65534:65534" + - /var/lib/mysql + volumeMounts: + - mountPath: /tmp + name: pod-tmp + - mountPath: /var/lib/mysql + name: mysql-data + containers: + - name: mariadb-backup + command: + - /bin/sh + args: + - -c + - >- + ( /tmp/start_verification_server.sh ) & + /tmp/backup_mariadb.sh + env: + - name: MARIADB_BACKUP_BASE_DIR + value: {{ .Values.conf.backup.base_path | quote }} + - name: MYSQL_BACKUP_MYSQLDUMP_OPTIONS + value: {{ .Values.conf.backup.mysqldump_options | quote }} + - name: MARIADB_LOCAL_BACKUP_DAYS_TO_KEEP + value: {{ .Values.conf.backup.days_to_keep | quote }} + - name: MARIADB_POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: REMOTE_BACKUP_ENABLED + value: "{{ .Values.conf.backup.remote_backup.enabled }}" +{{- if .Values.conf.backup.remote_backup.enabled }} + - name: MARIADB_REMOTE_BACKUP_DAYS_TO_KEEP + value: {{ .Values.conf.backup.remote_backup.days_to_keep | quote }} + - name: CONTAINER_NAME + value: {{ .Values.conf.backup.remote_backup.container_name | quote }} + - name: STORAGE_POLICY + value: "{{ .Values.conf.backup.remote_backup.storage_policy }}" + - name: NUMBER_OF_RETRIES_SEND_BACKUP_TO_REMOTE + value: {{ .Values.conf.backup.remote_backup.number_of_retries | quote }} + - name: MIN_DELAY_SEND_BACKUP_TO_REMOTE + value: {{ .Values.conf.backup.remote_backup.delay_range.min | quote }} + - name: MAX_DELAY_SEND_BACKUP_TO_REMOTE + value: {{ .Values.conf.backup.remote_backup.delay_range.max | quote }} + - name: THROTTLE_BACKUPS_ENABLED + value: "{{ .Values.conf.backup.remote_backup.throttle_backups.enabled }}" + - name: THROTTLE_LIMIT + value: {{ .Values.conf.backup.remote_backup.throttle_backups.sessions_limit | quote }} + - name: THROTTLE_LOCK_EXPIRE_AFTER + value: {{ .Values.conf.backup.remote_backup.throttle_backups.lock_expire_after | quote }} + - name: THROTTLE_RETRY_AFTER + value: {{ .Values.conf.backup.remote_backup.throttle_backups.retry_after | quote }} + - name: THROTTLE_CONTAINER_NAME + value: {{ .Values.conf.backup.remote_backup.throttle_backups.container_name | quote }} +{{- with $env := dict "ksUserSecret" $envAll.Values.secrets.identity.mariadb }} +{{- include "helm-toolkit.snippets.keystone_openrc_env_vars" $env | indent 16 }} +{{- end }} +{{- end }} +{{ tuple $envAll "mariadb_backup" | include "helm-toolkit.snippets.image" | indent 14 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.mariadb_backup | include "helm-toolkit.snippets.kubernetes_resources" | indent 14 }} +{{ dict "envAll" $envAll "application" "mariadb_backup" "container" "mariadb_backup" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 14 }} + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - mountPath: /tmp/backup_mariadb.sh + name: mariadb-bin + readOnly: true + subPath: backup_mariadb.sh + - mountPath: /tmp/backup_main.sh + name: mariadb-bin + readOnly: true + subPath: backup_main.sh + - mountPath: {{ .Values.conf.backup.base_path }} + name: mariadb-backup-dir + - name: mariadb-secrets + mountPath: /etc/mysql/admin_user.cnf + subPath: admin_user.cnf + readOnly: true +{{ dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.secrets.tls.oslo_db.server.internal "path" "/etc/mysql/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 16 }} + - name: mariadb-bin + mountPath: /tmp/start_verification_server.sh + readOnly: true + subPath: start_verification_server.sh + - name: mysql-data + mountPath: /var/lib/mysql + - name: var-run + mountPath: /run/mysqld + volumes: + - name: pod-tmp + emptyDir: {} + - name: mycnfd + emptyDir: {} + - name: var-run + emptyDir: {} + - name: mariadb-etc + configMap: + name: mariadb-etc + defaultMode: 0444 + - name: mysql-data + emptyDir: {} + - name: mariadb-secrets + secret: + secretName: mariadb-secrets + defaultMode: 420 + - configMap: + defaultMode: 365 + name: mariadb-bin + name: mariadb-bin + {{- if and .Values.volume.backup.enabled .Values.manifests.pvc_backup }} + - name: mariadb-backup-dir + persistentVolumeClaim: + claimName: mariadb-backup-data + {{- else }} + - hostPath: + path: {{ .Values.conf.backup.base_path }} + type: DirectoryOrCreate + name: mariadb-backup-dir + {{- end }} +{{ dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.secrets.tls.oslo_db.server.internal | include "helm-toolkit.snippets.tls_volume" | indent 12 }} +{{- end }} diff --git a/mariadb/templates/deployment-controller.yaml b/mariadb/templates/deployment-controller.yaml new file mode 100644 index 0000000000..a0fe46b2da --- /dev/null +++ b/mariadb/templates/deployment-controller.yaml @@ -0,0 +1,116 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.deployment_controller }} +{{- $envAll := . }} + +{{- $serviceAccountName := "mariadb-controller" }} +{{ tuple $envAll "controller" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $envAll.Release.Name }}-{{ $serviceAccountName }}-pod + namespace: {{ $envAll.Release.Namespace }} +rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - get + - list + - apiGroups: + - "" + resources: + - services + verbs: + - update + - patch + - get + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $envAll.Release.Name }}-{{ $serviceAccountName }}-pod + namespace: {{ $envAll.Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $envAll.Release.Name }}-{{ $serviceAccountName }}-pod +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +--- + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mariadb-controller + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "mariadb" "controller" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + replicas: {{ .Values.pod.replicas.controller }} + selector: + matchLabels: +{{ tuple $envAll "mariadb" "controller" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} +{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "mariadb" "controller" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + spec: + serviceAccountName: {{ $serviceAccountName }} +{{ dict "envAll" $envAll "application" "controller" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + affinity: +{{ tuple $envAll "mariadb" "controller" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} + nodeSelector: + {{ .Values.labels.controller.node_selector_key }}: {{ .Values.labels.controller.node_selector_value }} + initContainers: +{{ tuple $envAll "controller" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: controller +{{ tuple $envAll "mariadb_controller" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ dict "envAll" $envAll "application" "controller" "container" "controller" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.controller | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + command: + - /tmp/mariadb_controller.py + env: +{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.mariadb_controller | indent 12 }} + - name: MARIADB_CONTROLLER_PODS_NAMESPACE + value: {{ $envAll.Release.Namespace }} + - name: MARIADB_MASTER_SERVICE_NAME + value: {{ tuple "oslo_db" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - mountPath: /tmp/mariadb_controller.py + name: mariadb-bin + readOnly: true + subPath: mariadb_controller.py + volumes: + - name: pod-tmp + emptyDir: {} + - name: mariadb-bin + configMap: + name: mariadb-bin + defaultMode: 365 +{{- end }} diff --git a/mariadb/templates/exporter-configmap-bin.yaml b/mariadb/templates/exporter-configmap-bin.yaml new file mode 100644 index 0000000000..bcee0cd235 --- /dev/null +++ b/mariadb/templates/exporter-configmap-bin.yaml @@ -0,0 +1,27 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.monitoring.prometheus.configmap_bin .Values.monitoring.prometheus.enabled }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: mysql-exporter-bin +data: + create-mysql-user.sh: | +{{ tuple "bin/_prometheus-create-mysql-user.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + mysqld-exporter.sh: | +{{ tuple "bin/_prometheus-mysqld-exporter.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} diff --git a/mariadb/templates/exporter-job-create-user.yaml b/mariadb/templates/exporter-job-create-user.yaml new file mode 100644 index 0000000000..b2c1a1e38d --- /dev/null +++ b/mariadb/templates/exporter-job-create-user.yaml @@ -0,0 +1,92 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.monitoring.prometheus.job_user_create .Values.monitoring.prometheus.enabled }} +{{- $envAll := . }} + +{{- $serviceAccountName := "mariadb-exporter-create-sql-user" }} +{{ tuple $envAll "prometheus_create_mysql_user" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: mariadb-exporter-create-sql-user + labels: +{{ tuple $envAll "prometheus-mysql-exporter" "create-sql-user" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + backoffLimit: {{ .Values.jobs.exporter_create_sql_user.backoffLimit }} + template: + metadata: + labels: +{{ tuple $envAll "prometheus-mysql-exporter" "create-sql-user" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} +{{ dict "envAll" $envAll "podName" "create-sql-user" "containerNames" (list "init" "exporter-create-sql-user") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: + shareProcessNamespace: true + serviceAccountName: {{ $serviceAccountName }} +{{ dict "envAll" $envAll "application" "prometheus_create_mysql_user" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + activeDeadlineSeconds: {{ .Values.jobs.exporter_create_sql_user.activeDeadlineSeconds }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.prometheus_mysql_exporter.node_selector_key }}: {{ .Values.labels.prometheus_mysql_exporter.node_selector_value }} + initContainers: +{{ tuple $envAll "prometheus_create_mysql_user" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: exporter-create-sql-user +{{ tuple $envAll "prometheus_create_mysql_user" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ dict "envAll" $envAll "application" "prometheus_create_mysql_user" "container" "main" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.prometheus_create_mysql_user | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + command: + - /tmp/create-mysql-user.sh + env: + - name: EXPORTER_USER + valueFrom: + secretKeyRef: + name: mysql-exporter-secrets + key: EXPORTER_USER + - name: EXPORTER_PASSWORD + valueFrom: + secretKeyRef: + name: mysql-exporter-secrets + key: EXPORTER_PASSWORD +{{- if $envAll.Values.manifests.certificates }} + - name: MARIADB_X509 + value: "REQUIRE X509" +{{- end }} + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: mysql-exporter-bin + mountPath: /tmp/create-mysql-user.sh + subPath: create-mysql-user.sh + readOnly: true + - name: mariadb-secrets + mountPath: /etc/mysql/admin_user.cnf + subPath: admin_user.cnf + readOnly: true +{{ dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.secrets.tls.oslo_db.server.internal "path" "/etc/mysql/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: mysql-exporter-bin + configMap: + name: mysql-exporter-bin + defaultMode: 0555 + - name: mariadb-secrets + secret: + secretName: mariadb-secrets + defaultMode: 0444 +{{ dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.secrets.tls.oslo_db.server.internal | include "helm-toolkit.snippets.tls_volume" | indent 8 }} +{{- end }} diff --git a/mariadb/templates/exporter-secrets-etc.yaml b/mariadb/templates/exporter-secrets-etc.yaml new file mode 100644 index 0000000000..f45c2ca5a7 --- /dev/null +++ b/mariadb/templates/exporter-secrets-etc.yaml @@ -0,0 +1,33 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.monitoring.prometheus.secret_etc .Values.monitoring.prometheus.enabled }} +{{- $envAll := . }} + +{{- $exporter_user := .Values.endpoints.oslo_db.auth.exporter.username }} +{{- $exporter_password := .Values.endpoints.oslo_db.auth.exporter.password }} +{{- $db_host := "localhost" }} +{{- $data_source_name := printf "%s:%s@(%s)/" $exporter_user $exporter_password $db_host }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: mysql-exporter-secrets +type: Opaque +data: + DATA_SOURCE_NAME: {{ $data_source_name | b64enc }} + EXPORTER_USER: {{ .Values.endpoints.oslo_db.auth.exporter.username | b64enc }} + EXPORTER_PASSWORD: {{ .Values.endpoints.oslo_db.auth.exporter.password | b64enc }} + mysql_user.cnf: {{ tuple "secrets/_prometheus-exporter_user.cnf.tpl" . | include "helm-toolkit.utils.template" | b64enc }} +{{- end }} diff --git a/mariadb/templates/job-cluster-wait.yaml b/mariadb/templates/job-cluster-wait.yaml new file mode 100644 index 0000000000..30d96bf83b --- /dev/null +++ b/mariadb/templates/job-cluster-wait.yaml @@ -0,0 +1,125 @@ +{{/* +Copyright 2019 Mirantis inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.job_cluster_wait }} +{{- $envAll := . }} + +{{- $serviceAccountName := print .Release.Name "-cluster-wait" }} +{{ tuple $envAll "cluster_wait" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $envAll.Release.Name }}-{{ $serviceAccountName }}-pod + namespace: {{ $envAll.Release.Namespace }} +rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - update + - patch + - get + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $envAll.Release.Name }}-{{ $serviceAccountName }}-pod + namespace: {{ $envAll.Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $envAll.Release.Name }}-{{ $serviceAccountName }}-pod +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: "{{.Release.Name}}-cluster-wait" + labels: +{{ tuple $envAll "mariadb" "cluster-wait" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +spec: + backoffLimit: {{ .Values.jobs.cluster_wait.clusterCheckRetries }} + template: + metadata: + labels: +{{ tuple $envAll "mariadb" "cluster-wait" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "cluster_wait" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} + initContainers: +{{ tuple $envAll "cluster_wait" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: {{.Release.Name}}-mariadb-cluster-wait +{{ tuple $envAll "mariadb_scripted_test" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ dict "envAll" $envAll "application" "cluster_wait" "container" "mariadb_cluster_wait" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + env: + - name: MARIADB_HOST + value: {{ tuple "oslo_db" "internal" $envAll | include "helm-toolkit.endpoints.endpoint_host_lookup" }} + - name: MARIADB_REPLICAS + value: {{ .Values.pod.replicas.server | quote }} + - name: MARIADB_CLUSTER_CHECK_WAIT + value: {{ .Values.jobs.cluster_wait.clusterCheckWait | quote }} + - name: MARIADB_CLUSTER_STABILITY_COUNT + value: {{ .Values.jobs.cluster_wait.clusterStabilityCount | quote }} + - name: MARIADB_CLUSTER_STABILITY_WAIT + value: {{ .Values.jobs.cluster_wait.clusterStabilityWait | quote }} + - name: MARIADB_CLUSTER_STATE_CONFIGMAP + value: {{ printf "%s-%s" .Release.Name "mariadb-state" | quote }} + - name: MARIADB_CLUSTER_STATE_CONFIGMAP_NAMESPACE + value: {{ $envAll.Release.Namespace }} + - name: MARIADB_PASSWORD + valueFrom: + secretKeyRef: + name: mariadb-dbadmin-password + key: MYSQL_DBADMIN_PASSWORD + command: + - /tmp/mariadb-wait-for-cluster.py + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: mariadb-bin + mountPath: /tmp/mariadb-wait-for-cluster.py + subPath: mariadb-wait-for-cluster.py + readOnly: true + - name: mariadb-secrets + mountPath: /etc/mysql/admin_user.cnf + subPath: admin_user.cnf + readOnly: true +{{- dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.secrets.tls.oslo_db.server.internal "path" "/etc/mysql/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: mariadb-bin + configMap: + name: mariadb-bin + defaultMode: 0555 + - name: mariadb-secrets + secret: + secretName: mariadb-secrets + defaultMode: 0444 +{{- dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.secrets.tls.oslo_db.server.internal | include "helm-toolkit.snippets.tls_volume" | indent 8 }} +{{- end }} diff --git a/mariadb/templates/job-image-repo-sync.yaml b/mariadb/templates/job-image-repo-sync.yaml new file mode 100644 index 0000000000..2121a39753 --- /dev/null +++ b/mariadb/templates/job-image-repo-sync.yaml @@ -0,0 +1,21 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "mariadb" -}} +{{- if .Values.pod.tolerations.mariadb.enabled -}} +{{- $_ := set $imageRepoSyncJob "tolerationsEnabled" true -}} +{{- end -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/mariadb/templates/job-ks-user.yaml b/mariadb/templates/job-ks-user.yaml new file mode 100644 index 0000000000..fddf885835 --- /dev/null +++ b/mariadb/templates/job-ks-user.yaml @@ -0,0 +1,23 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.job_ks_user }} +{{- $backoffLimit := .Values.jobs.ks_user.backoffLimit }} +{{- $activeDeadlineSeconds := .Values.jobs.ks_user.activeDeadlineSeconds }} +{{- $ksUserJob := dict "envAll" . "serviceName" "mariadb" "configMapBin" "mariadb-bin" "backoffLimit" $backoffLimit "activeDeadlineSeconds" $activeDeadlineSeconds -}} +{{- if .Values.pod.tolerations.mariadb.enabled -}} +{{- $_ := set $ksUserJob "tolerationsEnabled" true -}} +{{- end -}} +{{ $ksUserJob | include "helm-toolkit.manifests.job_ks_user" }} +{{- end }} diff --git a/mariadb/templates/mariadb-backup-pvc.yaml b/mariadb/templates/mariadb-backup-pvc.yaml new file mode 100644 index 0000000000..c5b2174b30 --- /dev/null +++ b/mariadb/templates/mariadb-backup-pvc.yaml @@ -0,0 +1,28 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.volume.backup.enabled .Values.manifests.pvc_backup }} +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: mariadb-backup-data +spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: {{ .Values.volume.backup.size }} + storageClassName: {{ .Values.volume.backup.class_name }} +{{- end }} + diff --git a/mariadb/templates/network_policy.yaml b/mariadb/templates/network_policy.yaml new file mode 100644 index 0000000000..78ecc07bd0 --- /dev/null +++ b/mariadb/templates/network_policy.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{- if .Values.manifests.network_policy -}} +{{- $netpol_opts := dict "envAll" . "name" "application" "label" "mariadb" -}} +{{ $netpol_opts | include "helm-toolkit.manifests.kubernetes_network_policy" }} +{{- end -}} diff --git a/mariadb/templates/pdb-mariadb.yaml b/mariadb/templates/pdb-mariadb.yaml new file mode 100644 index 0000000000..163a432a29 --- /dev/null +++ b/mariadb/templates/pdb-mariadb.yaml @@ -0,0 +1,27 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.pdb_server }} +{{- $envAll := . }} +--- +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: mariadb-server +spec: + minAvailable: {{ .Values.pod.lifecycle.disruption_budget.mariadb.min_available }} + selector: + matchLabels: +{{ tuple $envAll "mariadb" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} +{{- end }} diff --git a/mariadb/templates/pod-test.yaml b/mariadb/templates/pod-test.yaml new file mode 100644 index 0000000000..c8b3c29c37 --- /dev/null +++ b/mariadb/templates/pod-test.yaml @@ -0,0 +1,86 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if (.Values.global).subchart_release_name }} +{{- $_ := set . "deployment_name" .Chart.Name }} +{{- else }} +{{- $_ := set . "deployment_name" .Release.Name }} +{{- end }} + +{{- if .Values.manifests.pod_test }} +{{- $envAll := . }} +{{- $dependencies := .Values.dependencies.static.tests }} + +{{- $serviceAccountName := print .deployment_name "-test" }} +{{ tuple $envAll "tests" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: v1 +kind: Pod +metadata: + name: "{{.deployment_name}}-test" + labels: +{{ tuple $envAll "mariadb" "test" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: + "helm.sh/hook": test-success + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +{{ dict "envAll" $envAll "podName" "mariadb-test" "containerNames" (list "init" "mariadb-test") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 4 }} +spec: + shareProcessNamespace: true + serviceAccountName: {{ $serviceAccountName }} +{{ dict "envAll" $envAll "application" "tests" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 2 }} +{{ if $envAll.Values.pod.tolerations.mariadb.enabled }} +{{ tuple $envAll "mariadb" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 2 }} +{{ end }} + nodeSelector: + {{ .Values.labels.test.node_selector_key }}: {{ .Values.labels.test.node_selector_value }} + restartPolicy: Never + initContainers: +{{ tuple $envAll "tests" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 4 }} + containers: + - name: mariadb-test +{{ dict "envAll" $envAll "application" "tests" "container" "test" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 6 }} +{{ tuple $envAll "scripted_test" | include "helm-toolkit.snippets.image" | indent 6 }} + command: + - /tmp/test.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: mariadb-bin + mountPath: /tmp/test.sh + subPath: test.sh + readOnly: true + - name: mariadb-secrets + mountPath: /etc/mysql/test-params.cnf + {{ if eq $envAll.Values.conf.tests.endpoint "internal" }} + subPath: admin_user_internal.cnf + {{ else if eq $envAll.Values.conf.tests.endpoint "direct" }} + subPath: admin_user.cnf + {{ else }} + {{ fail "Either 'direct' or 'internal' should be specified for .Values.conf.tests.endpoint" }} + {{ end }} + readOnly: true +{{ dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.secrets.tls.oslo_db.server.internal "path" "/etc/mysql/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 8 }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: mariadb-bin + configMap: + name: mariadb-bin + defaultMode: 0555 + - name: mariadb-secrets + secret: + secretName: mariadb-secrets + defaultMode: 0444 +{{ dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.secrets.tls.oslo_db.server.internal | include "helm-toolkit.snippets.tls_volume" | indent 4 }} +{{- end }} diff --git a/mariadb/templates/secret-backup-restore.yaml b/mariadb/templates/secret-backup-restore.yaml new file mode 100644 index 0000000000..1a37290b70 --- /dev/null +++ b/mariadb/templates/secret-backup-restore.yaml @@ -0,0 +1,35 @@ +{{/* +This manifest results a secret being created which has the key information +needed for backing up and restoring the Mariadb databases. +*/}} + +{{- if and .Values.conf.backup.enabled .Values.manifests.secret_backup_restore }} + +{{- $envAll := . }} +{{- $userClass := "backup_restore" }} +{{- $secretName := index $envAll.Values.secrets.mariadb $userClass }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} +type: Opaque +data: + BACKUP_ENABLED: {{ $envAll.Values.conf.backup.enabled | quote | b64enc }} + BACKUP_BASE_PATH: {{ $envAll.Values.conf.backup.base_path | b64enc }} + LOCAL_DAYS_TO_KEEP: {{ $envAll.Values.conf.backup.days_to_keep | quote | b64enc }} + MYSQLDUMP_OPTIONS: {{ $envAll.Values.conf.backup.mysqldump_options | b64enc }} + REMOTE_BACKUP_ENABLED: {{ $envAll.Values.conf.backup.remote_backup.enabled | quote | b64enc }} + REMOTE_BACKUP_CONTAINER: {{ $envAll.Values.conf.backup.remote_backup.container_name | b64enc }} + REMOTE_BACKUP_DAYS_TO_KEEP: {{ $envAll.Values.conf.backup.remote_backup.days_to_keep | quote | b64enc }} + REMOTE_BACKUP_STORAGE_POLICY: {{ $envAll.Values.conf.backup.remote_backup.storage_policy | b64enc }} + REMOTE_BACKUP_RETRIES: {{ $envAll.Values.conf.backup.remote_backup.number_of_retries | quote | b64enc }} + REMOTE_BACKUP_SEND_DELAY_MIN: {{ $envAll.Values.conf.backup.remote_backup.delay_range.min | quote | b64enc }} + REMOTE_BACKUP_SEND_DELAY_MAX: {{ $envAll.Values.conf.backup.remote_backup.delay_range.max | quote | b64enc }} + THROTTLE_BACKUPS_ENABLED: {{ $envAll.Values.conf.backup.remote_backup.throttle_backups.enabled | quote | b64enc }} + THROTTLE_LIMIT: {{ $envAll.Values.conf.backup.remote_backup.throttle_backups.sessions_limit | quote | b64enc }} + THROTTLE_LOCK_EXPIRE_AFTER: {{ $envAll.Values.conf.backup.remote_backup.throttle_backups.lock_expire_after | quote | b64enc }} + THROTTLE_RETRY_AFTER: {{ $envAll.Values.conf.backup.remote_backup.throttle_backups.retry_after | quote | b64enc }} + THROTTLE_CONTAINER_NAME: {{ $envAll.Values.conf.backup.remote_backup.throttle_backups.container_name | quote | b64enc }} +... +{{- end }} diff --git a/mariadb/templates/secret-dbadmin-password.yaml b/mariadb/templates/secret-dbadmin-password.yaml new file mode 100644 index 0000000000..c9f8c4e268 --- /dev/null +++ b/mariadb/templates/secret-dbadmin-password.yaml @@ -0,0 +1,25 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_dbadmin_password }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: mariadb-dbadmin-password +type: Opaque +data: + MYSQL_DBADMIN_PASSWORD: {{ .Values.endpoints.oslo_db.auth.admin.password | b64enc }} +{{- end }} diff --git a/mariadb/templates/secret-dbaudit-password.yaml b/mariadb/templates/secret-dbaudit-password.yaml new file mode 100644 index 0000000000..7733da7dd3 --- /dev/null +++ b/mariadb/templates/secret-dbaudit-password.yaml @@ -0,0 +1,25 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_dbaudit_password }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: mariadb-dbaudit-password +type: Opaque +data: + MYSQL_DBAUDIT_PASSWORD: {{ .Values.endpoints.oslo_db.auth.audit.password | b64enc }} +{{- end }} diff --git a/mariadb/templates/secret-registry.yaml b/mariadb/templates/secret-registry.yaml new file mode 100644 index 0000000000..da979b3223 --- /dev/null +++ b/mariadb/templates/secret-registry.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_registry .Values.endpoints.oci_image_registry.auth.enabled }} +{{ include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) }} +{{- end }} diff --git a/mariadb/templates/secret-rgw.yaml b/mariadb/templates/secret-rgw.yaml new file mode 100644 index 0000000000..086bba1b06 --- /dev/null +++ b/mariadb/templates/secret-rgw.yaml @@ -0,0 +1,78 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + + +This manifest results in two secrets being created: + 1) Keystone "mariadb" secret, which is needed to access the cluster + (remote or same cluster) for storing mariadb backups. If the + cluster is remote, the auth_url would be non-null. + 2) Keystone "admin" secret, which is needed to create the + "mariadb" keystone account mentioned above. This may not + be needed if the account is in a remote cluster (auth_url is non-null + in that case). +*/}} + +{{- if .Values.conf.backup.remote_backup.enabled }} + +{{- $envAll := . }} +{{- $userClass := "mariadb" }} +{{- $secretName := index $envAll.Values.secrets.identity $userClass }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} +type: Opaque +data: +{{- $identityClass := index .Values.endpoints.identity.auth $userClass }} +{{- if $identityClass.auth_url }} + OS_AUTH_URL: {{ $identityClass.auth_url | b64enc }} +{{- else }} + OS_AUTH_URL: {{ tuple "identity" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | b64enc }} +{{- end }} + OS_REGION_NAME: {{ $identityClass.region_name | b64enc }} + OS_INTERFACE: {{ $identityClass.interface | default "internal" | b64enc }} + OS_PROJECT_DOMAIN_NAME: {{ $identityClass.project_domain_name | b64enc }} + OS_PROJECT_NAME: {{ $identityClass.project_name | b64enc }} + OS_USER_DOMAIN_NAME: {{ $identityClass.user_domain_name | b64enc }} + OS_USERNAME: {{ $identityClass.username | b64enc }} + OS_PASSWORD: {{ $identityClass.password | b64enc }} + OS_DEFAULT_DOMAIN: {{ $identityClass.default_domain_id | default "default" | b64enc }} +... +{{- if .Values.manifests.job_ks_user }} +{{- $userClass := "admin" }} +{{- $secretName := index $envAll.Values.secrets.identity $userClass }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} +type: Opaque +data: +{{- $identityClass := index .Values.endpoints.identity.auth $userClass }} +{{- if $identityClass.auth_url }} + OS_AUTH_URL: {{ $identityClass.auth_url | b64enc }} +{{- else }} + OS_AUTH_URL: {{ tuple "identity" "internal" "api" $envAll | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | b64enc }} +{{- end }} + OS_REGION_NAME: {{ $identityClass.region_name | b64enc }} + OS_INTERFACE: {{ $identityClass.interface | default "internal" | b64enc }} + OS_PROJECT_DOMAIN_NAME: {{ $identityClass.project_domain_name | b64enc }} + OS_PROJECT_NAME: {{ $identityClass.project_name | b64enc }} + OS_USER_DOMAIN_NAME: {{ $identityClass.user_domain_name | b64enc }} + OS_USERNAME: {{ $identityClass.username | b64enc }} + OS_PASSWORD: {{ $identityClass.password | b64enc }} + OS_DEFAULT_DOMAIN: {{ $identityClass.default_domain_id | default "default" | b64enc }} +... +{{- end }} +{{- end }} diff --git a/mariadb/templates/secret-sst-password.yaml b/mariadb/templates/secret-sst-password.yaml new file mode 100644 index 0000000000..c49c0ff9b8 --- /dev/null +++ b/mariadb/templates/secret-sst-password.yaml @@ -0,0 +1,25 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_sst_password }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: mariadb-dbsst-password +type: Opaque +data: + MYSQL_DBSST_PASSWORD: {{ .Values.endpoints.oslo_db.auth.sst.password | b64enc }} +{{- end }} diff --git a/mariadb/templates/secrets-etc.yaml b/mariadb/templates/secrets-etc.yaml new file mode 100644 index 0000000000..9dac3eb1b0 --- /dev/null +++ b/mariadb/templates/secrets-etc.yaml @@ -0,0 +1,26 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_etc }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: mariadb-secrets +type: Opaque +data: + admin_user.cnf: {{ tuple "secrets/_admin_user.cnf.tpl" . | include "helm-toolkit.utils.template" | b64enc }} + admin_user_internal.cnf: {{ tuple "secrets/_admin_user_internal.cnf.tpl" . | include "helm-toolkit.utils.template" | b64enc }} +{{- end }} diff --git a/mariadb/templates/secrets/_admin_user.cnf.tpl b/mariadb/templates/secrets/_admin_user.cnf.tpl new file mode 100644 index 0000000000..0031a4bd7d --- /dev/null +++ b/mariadb/templates/secrets/_admin_user.cnf.tpl @@ -0,0 +1,24 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +[client] +user = {{ .Values.endpoints.oslo_db.auth.admin.username }} +password = {{ .Values.endpoints.oslo_db.auth.admin.password }} +host = {{ tuple "oslo_db" "direct" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +port = {{ tuple "oslo_db" "direct" "mysql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +{{- if .Values.manifests.certificates }} +ssl-ca = /etc/mysql/certs/ca.crt +ssl-key = /etc/mysql/certs/tls.key +ssl-cert = /etc/mysql/certs/tls.crt +{{- end }} diff --git a/mariadb/templates/secrets/_admin_user_internal.cnf.tpl b/mariadb/templates/secrets/_admin_user_internal.cnf.tpl new file mode 100644 index 0000000000..8bda8da013 --- /dev/null +++ b/mariadb/templates/secrets/_admin_user_internal.cnf.tpl @@ -0,0 +1,24 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +[client] +user = {{ .Values.endpoints.oslo_db.auth.admin.username }} +password = {{ .Values.endpoints.oslo_db.auth.admin.password }} +host = {{ tuple "oslo_db" "internal" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" }} +port = {{ tuple "oslo_db" "internal" "mysql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +{{- if .Values.manifests.certificates }} +ssl-ca = /etc/mysql/certs/ca.crt +ssl-key = /etc/mysql/certs/tls.key +ssl-cert = /etc/mysql/certs/tls.crt +{{- end }} diff --git a/mariadb/templates/secrets/_prometheus-exporter_user.cnf.tpl b/mariadb/templates/secrets/_prometheus-exporter_user.cnf.tpl new file mode 100644 index 0000000000..f09ed7f1bd --- /dev/null +++ b/mariadb/templates/secrets/_prometheus-exporter_user.cnf.tpl @@ -0,0 +1,24 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +[client] +user = {{ .Values.endpoints.oslo_db.auth.exporter.username }} +password = {{ .Values.endpoints.oslo_db.auth.exporter.password }} +host = localhost +port = {{ tuple "oslo_db" "direct" "mysql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +{{- if .Values.manifests.certificates }} +ssl-ca = /etc/mysql/certs/ca.crt +ssl-key = /etc/mysql/certs/tls.key +ssl-cert = /etc/mysql/certs/tls.crt +{{- end }} diff --git a/mariadb/templates/service-discovery.yaml b/mariadb/templates/service-discovery.yaml new file mode 100644 index 0000000000..d5efd3131c --- /dev/null +++ b/mariadb/templates/service-discovery.yaml @@ -0,0 +1,37 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.service_discovery }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "oslo_db" "discovery" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +spec: + ports: + - name: mysql + port: {{ tuple "oslo_db" "direct" "mysql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - name: wsrep + port: {{ tuple "oslo_db" "direct" "wsrep" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - name: ist + port: {{ tuple "oslo_db" "direct" "ist" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - name: sst + port: {{ tuple "oslo_db" "direct" "sst" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + clusterIP: None + publishNotReadyAddresses: false + selector: +{{ tuple $envAll "mariadb" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{ .Values.network.mariadb_discovery | include "helm-toolkit.snippets.service_params" | indent 2 }} +{{- end }} diff --git a/mariadb/templates/service-master.yaml b/mariadb/templates/service-master.yaml new file mode 100644 index 0000000000..e57824b18d --- /dev/null +++ b/mariadb/templates/service-master.yaml @@ -0,0 +1,30 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.service_master }} + +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "oslo_db" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +spec: + ports: + - name: mysql + port: {{ tuple "oslo_db" "direct" "mysql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + selector: +{{ tuple $envAll "mariadb" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{ .Values.network.mariadb_master | include "helm-toolkit.snippets.service_params" | indent 2 }} +{{- end }} diff --git a/mariadb/templates/service.yaml b/mariadb/templates/service.yaml new file mode 100644 index 0000000000..e68cbc49dd --- /dev/null +++ b/mariadb/templates/service.yaml @@ -0,0 +1,29 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.service }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "oslo_db" "direct" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +spec: + ports: + - name: mysql + port: {{ tuple "oslo_db" "direct" "mysql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + selector: +{{ tuple $envAll "mariadb" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{ .Values.network.mariadb | include "helm-toolkit.snippets.service_params" | indent 2 }} +{{- end }} diff --git a/mariadb/templates/statefulset.yaml b/mariadb/templates/statefulset.yaml new file mode 100644 index 0000000000..889ff71e37 --- /dev/null +++ b/mariadb/templates/statefulset.yaml @@ -0,0 +1,370 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "mariadbReadinessProbe" }} +exec: + command: + - /tmp/health.sh + - -t + - readiness + - -d + - {{ .Values.pod.probes.server.mariadb.readiness.disk_usage_percent | quote }} +{{- end }} +{{- define "mariadbLivenessProbe" }} +exec: + command: + - /tmp/health.sh + - -t + - liveness +{{- end }} +{{- define "exporterProbeTemplate" }} +httpGet: + path: /metrics + port: {{ tuple "prometheus_mysql_exporter" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +{{- end }} + +{{- if (.Values.global).subchart_release_name }} +{{- $_ := set . "deployment_name" .Chart.Name }} +{{- else }} +{{- $_ := set . "deployment_name" .Release.Name }} +{{- end }} + +{{- if .Values.manifests.statefulset }} +{{- $envAll := . }} + +{{- $serviceAccountName := printf "%s-%s" .deployment_name "mariadb" }} +{{ tuple $envAll "mariadb" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - apiGroups: + - "" + resourceNames: + - {{ printf "%s-%s" .deployment_name "mariadb-state" | quote }} + resources: + - configmaps + verbs: + - get + - patch + - apiGroups: + - "" + resourceNames: + - {{ tuple "oslo_db" "direct" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + resources: + - endpoints + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + # NOTE(portdirect): the statefulset name must match the POD_NAME_PREFIX env var for discovery to work + name: {{ tuple "oslo_db" "direct" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} + mariadb-dbadmin-password-hash: {{ tuple "secret-dbadmin-password.yaml" . | include "helm-toolkit.utils.hash" }} + mariadb-sst-password-hash: {{ tuple "secret-dbadmin-password.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-bin-exporter-hash: {{ tuple "exporter-configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + labels: +{{ tuple $envAll "mariadb" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + serviceName: "{{ tuple "oslo_db" "discovery" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}" + podManagementPolicy: "Parallel" + replicas: {{ .Values.pod.replicas.server }} + selector: + matchLabels: +{{ tuple $envAll "mariadb" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} + template: + metadata: + labels: +{{ tuple $envAll "mariadb" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} + mariadb-dbadmin-password-hash: {{ tuple "secret-dbadmin-password.yaml" . | include "helm-toolkit.utils.hash" }} + mariadb-sst-password-hash: {{ tuple "secret-dbadmin-password.yaml" . | include "helm-toolkit.utils.hash" }} +{{ dict "envAll" $envAll "podName" "mariadb-server" "containerNames" (list "init" "mariadb-perms" "mariadb") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: + shareProcessNamespace: true + serviceAccountName: {{ $serviceAccountName }} +{{ dict "envAll" $envAll "application" "server" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + affinity: +{{ tuple $envAll "mariadb" "server" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} +{{ if $envAll.Values.pod.tolerations.mariadb.enabled }} +{{ tuple $envAll "mariadb" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }} +{{ end }} + terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.server.timeout }} + nodeSelector: + {{ .Values.labels.server.node_selector_key }}: {{ .Values.labels.server.node_selector_value }} + initContainers: +{{ tuple $envAll "mariadb" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} +{{- if .Values.volume.chown_on_start }} + - name: mariadb-perms +{{ tuple $envAll "mariadb" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ dict "envAll" $envAll "application" "server" "container" "perms" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + command: ["/bin/sh", "-c"] + args: + - set -xe; + /bin/chown -R "mysql:mysql" /var/lib/mysql; + /bin/chmod 700 /var/lib/mysql; + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: mysql-data + mountPath: /var/lib/mysql +{{- end }} + containers: + - name: mariadb +{{ tuple $envAll "mariadb" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ dict "envAll" $envAll "application" "server" "container" "mariadb" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + env: + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + {{- if $envAll.Values.manifests.certificates }} + - name: MARIADB_X509 + value: "REQUIRE X509" + {{- end }} + - name: MARIADB_REPLICAS + value: {{ .Values.pod.replicas.server | quote }} + - name: POD_NAME_PREFIX + value: {{ tuple "oslo_db" "direct" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + - name: DISCOVERY_DOMAIN + value: {{ tuple "oslo_db" "discovery" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" }} + - name: DIRECT_SVC_NAME + value: {{ tuple "oslo_db" "direct" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + - name: WSREP_PORT + value: {{ tuple "oslo_db" "direct" "wsrep" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: STATE_CONFIGMAP + value: {{ printf "%s-%s" .deployment_name "mariadb-state" | quote }} + - name: MYSQL_DBADMIN_USERNAME + value: {{ .Values.endpoints.oslo_db.auth.admin.username }} + - name: MYSQL_DBADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: mariadb-dbadmin-password + key: MYSQL_DBADMIN_PASSWORD + - name: MYSQL_DBSST_USERNAME + value: {{ .Values.endpoints.oslo_db.auth.sst.username }} + - name: MYSQL_DBSST_PASSWORD + valueFrom: + secretKeyRef: + name: mariadb-dbsst-password + key: MYSQL_DBSST_PASSWORD + {{- if .Values.manifests.secret_dbaudit_password }} + - name: MYSQL_DBAUDIT_USERNAME + value: {{ .Values.endpoints.oslo_db.auth.audit.username }} + - name: MYSQL_DBAUDIT_PASSWORD + valueFrom: + secretKeyRef: + name: mariadb-dbaudit-password + key: MYSQL_DBAUDIT_PASSWORD + {{- end }} + - name: MYSQL_HISTFILE + value: {{ .Values.conf.database.mysql_histfile }} + - name: CLUSTER_LEADER_TTL + value: {{ .Values.conf.galera.cluster_leader_ttl | quote }} + ports: + - name: mysql + protocol: TCP + containerPort: {{ tuple "oslo_db" "direct" "mysql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - name: wsrep + protocol: TCP + containerPort: {{ tuple "oslo_db" "direct" "wsrep" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - name: ist + protocol: TCP + containerPort: {{ tuple "oslo_db" "direct" "ist" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - name: sst + protocol: TCP + containerPort: {{ tuple "oslo_db" "direct" "sst" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + command: + - /tmp/start.py +{{ dict "envAll" . "component" "server" "container" "mariadb" "type" "readiness" "probeTemplate" (include "mariadbReadinessProbe" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }} +{{ dict "envAll" . "component" "server" "container" "mariadb" "type" "liveness" "probeTemplate" (include "mariadbLivenessProbe" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }} + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: var-run + mountPath: /var/run/mysqld + - name: mycnfd + mountPath: /etc/mysql/conf.d + - name: mariadb-bin + mountPath: /tmp/start.py + subPath: start.py + readOnly: true + - name: mariadb-bin + mountPath: /tmp/stop.sh + subPath: stop.sh + readOnly: true + - name: mariadb-bin + mountPath: /tmp/health.sh + subPath: health.sh + readOnly: true + - name: mariadb-etc + mountPath: /etc/mysql/my.cnf + subPath: my.cnf + readOnly: true + - name: mariadb-etc + mountPath: /etc/mysql/conf.d/00-base.cnf + subPath: 00-base.cnf + readOnly: true + {{- if .Values.conf.database.config_override }} + - name: mariadb-etc + mountPath: /etc/mysql/conf.d/20-override.cnf + subPath: 20-override.cnf + readOnly: true + {{- end }} + - name: mariadb-etc + mountPath: /etc/mysql/conf.d/99-force.cnf + subPath: 99-force.cnf + readOnly: true + - name: mariadb-secrets + mountPath: /etc/mysql/admin_user.cnf + subPath: admin_user.cnf + readOnly: true + - name: mysql-data + mountPath: /var/lib/mysql +{{ dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.secrets.tls.oslo_db.server.internal "path" "/etc/mysql/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }} +{{- if .Values.monitoring.prometheus.enabled }} + - name: mysql-exporter +{{ tuple $envAll "prometheus_mysql_exporter" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ dict "envAll" $envAll "application" "server" "container" "exporter" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.prometheus_mysql_exporter | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" . "component" "server" "container" "mariadb_exporter" "type" "readiness" "probeTemplate" (include "exporterProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }} +{{ dict "envAll" . "component" "server" "container" "mariadb_exporter" "type" "liveness" "probeTemplate" (include "exporterProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }} + command: + - /tmp/mysqld-exporter.sh + ports: + - name: metrics + containerPort: {{ tuple "prometheus_mysql_exporter" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + env: + - name: EXPORTER_USER + valueFrom: + secretKeyRef: + name: mysql-exporter-secrets + key: EXPORTER_USER + - name: EXPORTER_PASSWORD + valueFrom: + secretKeyRef: + name: mysql-exporter-secrets + key: EXPORTER_PASSWORD + - name: DATA_SOURCE_NAME + valueFrom: + secretKeyRef: + name: mysql-exporter-secrets + key: DATA_SOURCE_NAME + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: LISTEN_PORT + value: {{ tuple "prometheus_mysql_exporter" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: TELEMETRY_PATH + value: {{ tuple "prometheus_mysql_exporter" "internal" "metrics" . | include "helm-toolkit.endpoints.keystone_endpoint_path_lookup" | quote }} + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: mysql-exporter-secrets + mountPath: /etc/mysql/mysql_user.cnf + subPath: mysql_user.cnf + readOnly: true + - name: mysql-exporter-bin + mountPath: /tmp/mysqld-exporter.sh + subPath: mysqld-exporter.sh + readOnly: true +{{- end }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: mycnfd + emptyDir: {} + - name: var-run + emptyDir: {} + - name: mariadb-bin + configMap: + name: mariadb-bin + defaultMode: 0555 + - name: mariadb-etc + configMap: + name: mariadb-etc + defaultMode: 0444 + - name: mariadb-secrets + secret: + secretName: mariadb-secrets + defaultMode: 0444 +{{ dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.secrets.tls.oslo_db.server.internal | include "helm-toolkit.snippets.tls_volume" | indent 8 }} + {{- if not .Values.volume.enabled }} + - name: mysql-data + {{- if .Values.volume.use_local_path_for_single_pod_cluster.enabled }} + hostPath: + path: {{ .Values.volume.use_local_path_for_single_pod_cluster.host_path }} + type: DirectoryOrCreate + {{- else }} + emptyDir: {} + {{- end }} + {{- end }} +{{- if .Values.monitoring.prometheus.enabled }} + - name: mysql-exporter-secrets + secret: + secretName: mysql-exporter-secrets + defaultMode: 0444 + - name: mysql-exporter-bin + configMap: + name: mysql-exporter-bin + defaultMode: 0555 +{{- end }} +{{- if .Values.volume.enabled }} + volumeClaimTemplates: + - metadata: + name: mysql-data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: {{ .Values.volume.size }} + {{- if ne .Values.volume.class_name "default" }} + storageClassName: {{ .Values.volume.class_name }} + {{- end }} +{{- end }} +{{- end }} diff --git a/mariadb/values.yaml b/mariadb/values.yaml new file mode 100644 index 0000000000..4fa5fb50d1 --- /dev/null +++ b/mariadb/values.yaml @@ -0,0 +1,712 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for mariadb. +# This is a YAML-formatted file. +# Declare name/value pairs to be passed into your templates. +# name: value + +--- +release_group: null + +images: + tags: + mariadb: docker.io/openstackhelm/mariadb:latest-ubuntu_jammy + prometheus_create_mysql_user: docker.io/library/mariadb:10.5.9-focal + prometheus_mysql_exporter: docker.io/prom/mysqld-exporter:v0.12.1 + prometheus_mysql_exporter_helm_tests: docker.io/openstackhelm/heat:wallaby-ubuntu_focal + dep_check: quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_jammy + image_repo_sync: docker.io/library/docker:17.07.0 + mariadb_backup: quay.io/airshipit/porthole-mysqlclient-utility:latest-ubuntu_jammy + ks_user: docker.io/openstackhelm/heat:wallaby-ubuntu_focal + scripted_test: docker.io/openstackhelm/mariadb:ubuntu_focal-20210415 + mariadb_controller: docker.io/openstackhelm/mariadb:latest-ubuntu_jammy + pull_policy: "IfNotPresent" + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +labels: + server: + node_selector_key: openstack-control-plane + node_selector_value: enabled + prometheus_mysql_exporter: + node_selector_key: openstack-control-plane + node_selector_value: enabled + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + test: + node_selector_key: openstack-control-plane + node_selector_value: enabled + controller: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +pod: + env: + mariadb_controller: + MARIADB_CONTROLLER_DEBUG: 0 + MARIADB_CONTROLLER_CHECK_PODS_DELAY: 10 + MARIADB_CONTROLLER_PYKUBE_REQUEST_TIMEOUT: 60 + probes: + server: + mariadb: + readiness: + enabled: true + disk_usage_percent: 99 + params: + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 15 + liveness: + enabled: true + params: + initialDelaySeconds: 120 + periodSeconds: 30 + timeoutSeconds: 15 + mariadb_exporter: + readiness: + enabled: true + params: + initialDelaySeconds: 5 + periodSeconds: 60 + timeoutSeconds: 10 + liveness: + enabled: true + params: + initialDelaySeconds: 15 + periodSeconds: 60 + timeoutSeconds: 10 + security_context: + server: + pod: + runAsUser: 999 + container: + perms: + runAsUser: 0 + readOnlyRootFilesystem: true + mariadb: + runAsUser: 999 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + prometheus_mysql_exporter: + pod: + runAsUser: 99 + container: + exporter: + runAsUser: 99 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + prometheus_create_mysql_user: + pod: + runAsUser: 0 + container: + main: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + mariadb_backup: + pod: + runAsUser: 65534 + container: + backup_perms: + runAsUser: 0 + readOnlyRootFilesystem: true + verify_perms: + runAsUser: 0 + readOnlyRootFilesystem: true + mariadb_backup: + runAsUser: 65534 + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + tests: + pod: + runAsUser: 999 + container: + test: + runAsUser: 999 + readOnlyRootFilesystem: true + controller: + pod: + runAsUser: 65534 + container: + controller: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + cluster_wait: + pod: + runAsUser: 65534 + runAsNonRoot: true + container: + mariadb_cluster_wait: + allowPrivilegeEscalation: false + capabilities: + drop: + - ALL + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + weight: + default: 10 + tolerations: + mariadb: + enabled: false + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + replicas: + server: 3 + controller: 1 + lifecycle: + upgrades: + deployments: + revision_history: 3 + pod_replacement_strategy: RollingUpdate + rolling_update: + max_unavailable: 1 + max_surge: 3 + termination_grace_period: + server: + timeout: 600 + disruption_budget: + mariadb: + min_available: 0 + resources: + enabled: false + server: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + jobs: + tests: + limits: + memory: "1024Mi" + cpu: "2000m" + requests: + memory: "128Mi" + cpu: "100m" + prometheus_create_mysql_user: + limits: + memory: "1024Mi" + cpu: "2000m" + requests: + memory: "128Mi" + cpu: "100m" + image_repo_sync: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + mariadb_backup: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + ks_user: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - mariadb-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + mariadb_backup: + jobs: + - mariadb-ks-user + services: + - endpoint: internal + service: oslo_db + prometheus_create_mysql_user: + services: + - endpoint: internal + service: oslo_db + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + tests: + services: + - endpoint: internal + service: oslo_db + controller: + services: null + cluster_wait: + services: + - endpoint: internal + service: oslo_db +volume: + # this value is used for single pod deployments of mariadb to prevent losing all data + # if the pod is restarted + use_local_path_for_single_pod_cluster: + enabled: false + host_path: "/tmp/mysql-data" + chown_on_start: true + enabled: true + class_name: general + size: 5Gi + backup: + enabled: true + class_name: general + size: 5Gi + +jobs: + cluster_wait: + clusterCheckWait: 30 + clusterCheckRetries: 30 + clusterStabilityCount: 30 + clusterStabilityWait: 4 + exporter_create_sql_user: + backoffLimit: 87600 + activeDeadlineSeconds: 3600 + mariadb_backup: + # activeDeadlineSeconds == 0 means no deadline + activeDeadlineSeconds: 0 + backoffLimit: 6 + cron: "0 0 * * *" + history: + success: 3 + failed: 1 + ks_user: + # activeDeadlineSeconds == 0 means no deadline + activeDeadlineSeconds: 0 + backoffLimit: 6 + +conf: + tests: + # This may either be: + # * direct: which will hit the backends directly via a k8s service ip + # Note, deadlocks and failure are to be expected with concurrency if + # hitting the `direct` endpoint. + endpoint: internal + # This is a list of tuning params passed to mysqlslap: + params: + - --auto-generate-sql + - --concurrency=100 + - --number-of-queries=1000 + - --number-char-cols=1 + - --number-int-cols=1 + mariadb_server: + setup_wait: + iteration: 30 + duration: 5 + backup: + enabled: false + base_path: /var/backup + validateData: + ageOffset: 120 + mysqldump_options: > + --single-transaction --quick --add-drop-database + --add-drop-table --add-locks --databases + days_to_keep: 3 + remote_backup: + enabled: false + container_name: mariadb + days_to_keep: 14 + storage_policy: default-placement + number_of_retries: 5 + delay_range: + min: 30 + max: 60 + throttle_backups: + enabled: false + sessions_limit: 480 + lock_expire_after: 7200 + retry_after: 3600 + container_name: throttle-backups-manager + galera: + cluster_leader_ttl: 60 + database: + mysql_histfile: "/dev/null" + my: | + [mysqld] + datadir=/var/lib/mysql + basedir=/usr + ignore-db-dirs=lost+found + + [client-server] + !includedir /etc/mysql/conf.d/ + 00_base: | + [mysqld] + # Charset + character_set_server=utf8 + collation_server=utf8_general_ci + skip-character-set-client-handshake + + # Logging + slow_query_log=off + slow_query_log_file=/var/log/mysql/mariadb-slow.log + log_warnings=2 + + # General logging has huge performance penalty therefore is disabled by default + general_log=off + general_log_file=/var/log/mysql/mariadb-error.log + + long_query_time=3 + log_queries_not_using_indexes=on + + # Networking + bind_address=0.0.0.0 + port={{ tuple "oslo_db" "direct" "mysql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + + # When a client connects, the server will perform hostname resolution, + # and when DNS is slow, establishing the connection will become slow as well. + # It is therefore recommended to start the server with skip-name-resolve to + # disable all DNS lookups. The only limitation is that the GRANT statements + # must then use IP addresses only. + skip_name_resolve + + # Tuning + user=mysql + max_allowed_packet=256M + open_files_limit=10240 + max_connections=8192 + max-connect-errors=1000000 + + # General security settings + # Reference: https://dev.mysql.com/doc/mysql-security-excerpt/8.0/en/general-security-issues.html + # secure_file_priv is set to '/home' because it is read-only, which will + # disable this feature completely. + secure_file_priv=/home + local_infile=0 + symbolic_links=0 + sql_mode="STRICT_ALL_TABLES,STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION" + + + ## Generally, it is unwise to set the query cache to be larger than 64-128M + ## as the costs associated with maintaining the cache outweigh the performance + ## gains. + ## The query cache is a well known bottleneck that can be seen even when + ## concurrency is moderate. The best option is to disable it from day 1 + ## by setting query_cache_size=0 (now the default on MySQL 5.6) + ## and to use other ways to speed up read queries: good indexing, adding + ## replicas to spread the read load or using an external cache. + query_cache_size=0 + query_cache_type=0 + + sync_binlog=0 + thread_cache_size=16 + table_open_cache=2048 + table_definition_cache=1024 + + # + # InnoDB + # + # The buffer pool is where data and indexes are cached: having it as large as possible + # will ensure you use memory and not disks for most read operations. + # Typical values are 50..75% of available RAM. + # TODO(tomasz.paszkowski): This needs to by dynamic based on available RAM. + innodb_buffer_pool_size=1024M + innodb_doublewrite=0 + innodb_file_format=Barracuda + innodb_file_per_table=1 + innodb_flush_method=O_DIRECT + innodb_io_capacity=500 + innodb_locks_unsafe_for_binlog=1 + innodb_log_file_size=128M + innodb_old_blocks_time=1000 + innodb_read_io_threads=8 + innodb_write_io_threads=8 + + # Clustering + binlog_format=ROW + default-storage-engine=InnoDB + innodb_autoinc_lock_mode=2 + innodb_flush_log_at_trx_commit=2 + wsrep_cluster_name={{ tuple "oslo_db" "direct" . | include "helm-toolkit.endpoints.hostname_namespaced_endpoint_lookup" | replace "." "_" }} + wsrep_on=1 + wsrep_provider=/usr/lib/galera/libgalera_smm.so + wsrep_provider_options="evs.suspect_timeout=PT30S; gmcast.peer_timeout=PT15S; gmcast.listen_addr=tcp://0.0.0.0:{{ tuple "oslo_db" "direct" "wsrep" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}" + wsrep_slave_threads=12 + wsrep_sst_auth={{ .Values.endpoints.oslo_db.auth.sst.username }}:{{ .Values.endpoints.oslo_db.auth.sst.password }} + wsrep_sst_method=mariabackup + + {{ if .Values.manifests.certificates }} + # TLS + ssl_ca=/etc/mysql/certs/ca.crt + ssl_key=/etc/mysql/certs/tls.key + ssl_cert=/etc/mysql/certs/tls.crt + # tls_version = TLSv1.2,TLSv1.3 + {{ end }} + + + [mysqldump] + max-allowed-packet=16M + + [client] + default_character_set=utf8 + protocol=tcp + port={{ tuple "oslo_db" "direct" "mysql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + {{ if .Values.manifests.certificates }} + # TLS + ssl_ca=/etc/mysql/certs/ca.crt + ssl_key=/etc/mysql/certs/tls.key + ssl_cert=/etc/mysql/certs/tls.crt + # tls_version = TLSv1.2,TLSv1.3 + ssl-verify-server-cert + {{ end }} + + config_override: null + # Any configuration here will override the base config. + # config_override: |- + # [mysqld] + # wsrep_slave_threads=1 + 99_force: | + [mysqld] + datadir=/var/lib/mysql + tmpdir=/tmp + +monitoring: + prometheus: + enabled: false + mysqld_exporter: + scrape: true + +secrets: + identity: + admin: keystone-admin-user + mariadb: mariadb-backup-user + mariadb: + backup_restore: mariadb-backup-restore + oci_image_registry: + mariadb: mariadb-oci-image-registry-key + tls: + oslo_db: + server: + public: mariadb-tls-server + internal: mariadb-tls-direct + +# typically overridden by environmental +# values, but should include all endpoints +# required by this chart +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + mariadb: + username: mariadb + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + monitoring: + name: prometheus + namespace: null + hosts: + default: prom-metrics + public: prometheus + host_fqdn_override: + default: null + path: + default: null + scheme: + default: 'http' + port: + api: + default: 9090 + public: 80 + prometheus_mysql_exporter: + namespace: null + hosts: + default: mysql-exporter + host_fqdn_override: + default: null + path: + default: /metrics + scheme: + default: 'http' + port: + metrics: + default: 9104 + oslo_db: + namespace: null + auth: + admin: + username: root + password: password + sst: + username: sst + password: password + audit: + username: audit + password: password + exporter: + username: exporter + password: password + hosts: + default: mariadb + direct: mariadb-server + discovery: mariadb-discovery + host_fqdn_override: + default: null + path: null + scheme: mysql+pymysql + port: + mysql: + default: 3306 + wsrep: + default: 4567 + ist: + default: 4568 + sst: + default: 4444 + kube_dns: + namespace: kube-system + name: kubernetes-dns + hosts: + default: kube-dns + host_fqdn_override: + default: null + path: + default: null + scheme: http + port: + dns_tcp: + default: 53 + dns: + default: 53 + protocol: UDP + identity: + name: backup-storage-auth + namespace: openstack + auth: + admin: + # Auth URL of null indicates local authentication + # HTK will form the URL unless specified here + auth_url: null + region_name: RegionOne + username: admin + password: password + project_name: admin + user_domain_name: default + project_domain_name: default + mariadb: + # Auth URL of null indicates local authentication + # HTK will form the URL unless specified here + auth_url: null + role: admin + region_name: RegionOne + username: mariadb-backup-user + password: password + project_name: service + user_domain_name: service + project_domain_name: service + hosts: + default: keystone + internal: keystone-api + host_fqdn_override: + default: null + path: + default: /v3 + scheme: + default: 'http' + port: + api: + default: 80 + internal: 5000 + +network: + mariadb: {} + mariadb_discovery: {} + mariadb_master: {} + +network_policy: + mariadb: + ingress: + - {} + egress: + - {} + prometheus-mysql-exporter: + ingress: + - {} + egress: + - {} + +# Helm hook breaks for helm2. +# Set helm3_hook: false in case helm2 is used. +helm3_hook: true + +manifests: + certificates: false + configmap_bin: true + configmap_etc: true + configmap_services_tcp: true + job_image_repo_sync: true + cron_job_mariadb_backup: false + job_ks_user: false + pvc_backup: false + monitoring: + prometheus: + configmap_bin: true + job_user_create: true + secret_etc: true + pdb_server: true + network_policy: false + pod_test: true + secret_dbadmin_password: true + secret_sst_password: true + secret_dbaudit_password: true + secret_backup_restore: false + secret_etc: true + secret_registry: true + service_discovery: true + service_error: false + service: true + statefulset: true + deployment_controller: true + service_master: true + job_cluster_wait: false +... diff --git a/memcached/Chart.yaml b/memcached/Chart.yaml new file mode 100644 index 0000000000..16f16464bc --- /dev/null +++ b/memcached/Chart.yaml @@ -0,0 +1,24 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: v1.5.5 +description: OpenStack-Helm Memcached +name: memcached +version: 2024.2.0 +home: https://github.com/memcached/memcached +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit + version: ">= 0.1.0" +... diff --git a/memcached/templates/bin/_memcached-exporter.sh.tpl b/memcached/templates/bin/_memcached-exporter.sh.tpl new file mode 100644 index 0000000000..08b4d919e9 --- /dev/null +++ b/memcached/templates/bin/_memcached-exporter.sh.tpl @@ -0,0 +1,28 @@ +#!/bin/sh + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +COMMAND="${@:-start}" + +function start () { + exec /bin/memcached_exporter --memcached.address "$MEMCACHED_HOST:$MEMCACHED_PORT" +} + +function stop () { + kill -TERM 1 +} + +$COMMAND diff --git a/memcached/templates/bin/_memcached.sh.tpl b/memcached/templates/bin/_memcached.sh.tpl new file mode 100644 index 0000000000..c727c286db --- /dev/null +++ b/memcached/templates/bin/_memcached.sh.tpl @@ -0,0 +1,27 @@ +#!/bin/sh + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +memcached --version +exec memcached -v \ + -p ${MEMCACHED_PORT} \ + -U 0 \ +{{- if not .Values.conf.memcached.stats_cachedump.enabled }} + -X \ +{{- end }} + -c ${MEMCACHED_MAX_CONNECTIONS} \ + -m ${MEMCACHED_MEMORY} diff --git a/memcached/templates/configmap-apparmor.yaml b/memcached/templates/configmap-apparmor.yaml new file mode 100644 index 0000000000..0a06bf7cc0 --- /dev/null +++ b/memcached/templates/configmap-apparmor.yaml @@ -0,0 +1,15 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- dict "envAll" . "component" "memcached" | include "helm-toolkit.snippets.kubernetes_apparmor_configmap" }} diff --git a/memcached/templates/configmap-bin.yaml b/memcached/templates/configmap-bin.yaml new file mode 100644 index 0000000000..f14bd242e0 --- /dev/null +++ b/memcached/templates/configmap-bin.yaml @@ -0,0 +1,38 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if (.Values.global).subchart_release_name }} +{{- $_ := set . "deployment_name" .Chart.Name }} +{{- else }} +{{- $_ := set . "deployment_name" .Release.Name }} +{{- end }} + +{{- if .Values.manifests.configmap_bin }} +{{- $envAll := . }} +{{- $configMapBinName := printf "%s-%s" $envAll.deployment_name "memcached-bin" }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $configMapBinName }} +data: +{{- if .Values.images.local_registry.active }} + image-repo-sync.sh: | +{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} +{{- end }} + memcached.sh: | +{{ tuple "bin/_memcached.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + memcached-exporter.sh: | +{{ tuple "bin/_memcached-exporter.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} diff --git a/memcached/templates/job-image-repo-sync.yaml b/memcached/templates/job-image-repo-sync.yaml new file mode 100644 index 0000000000..ae519ff026 --- /dev/null +++ b/memcached/templates/job-image-repo-sync.yaml @@ -0,0 +1,21 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "memcached" -}} +{{- if .Values.pod.tolerations.memcached.enabled -}} +{{- $_ := set $imageRepoSyncJob "tolerationsEnabled" true -}} +{{- end -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/memcached/templates/network_policy.yaml b/memcached/templates/network_policy.yaml new file mode 100644 index 0000000000..9beab0d75e --- /dev/null +++ b/memcached/templates/network_policy.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{- if .Values.manifests.network_policy -}} +{{- $netpol_opts := dict "envAll" . "name" "application" "label" "memcached" -}} +{{ $netpol_opts | include "helm-toolkit.manifests.kubernetes_network_policy" }} +{{- end -}} diff --git a/memcached/templates/secret-registry.yaml b/memcached/templates/secret-registry.yaml new file mode 100644 index 0000000000..da979b3223 --- /dev/null +++ b/memcached/templates/secret-registry.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_registry .Values.endpoints.oci_image_registry.auth.enabled }} +{{ include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) }} +{{- end }} diff --git a/memcached/templates/service.yaml b/memcached/templates/service.yaml new file mode 100644 index 0000000000..982647b1b4 --- /dev/null +++ b/memcached/templates/service.yaml @@ -0,0 +1,38 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.service }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "oslo_cache" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +spec: + sessionAffinity: ClientIP + ports: + - name: memcache + port: {{ tuple "oslo_cache" "internal" "memcache" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +{{/* +Keep exporter port here to be able to scrape over endpoints. +https://prometheus.io/docs/prometheus/latest/configuration/configuration/#endpoints +*/}} +{{- if .Values.monitoring.prometheus.enabled }} + - name: metrics + port: {{ tuple "oslo_cache" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +{{- end }} + selector: +{{ tuple $envAll "memcached" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{ .Values.network.memcached | include "helm-toolkit.snippets.service_params" | indent 2 }} +{{- end }} diff --git a/memcached/templates/statefulset.yaml b/memcached/templates/statefulset.yaml new file mode 100644 index 0000000000..77692d1bb4 --- /dev/null +++ b/memcached/templates/statefulset.yaml @@ -0,0 +1,137 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if (.Values.global).subchart_release_name }} +{{- $_ := set . "deployment_name" .Chart.Name }} +{{- else }} +{{- $_ := set . "deployment_name" .Release.Name }} +{{- end }} + +{{- define "memcachedProbeTemplate" }} +tcpSocket: + port: {{ tuple "oslo_cache" "internal" "memcache" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +{{- end }} + +{{- define "exporterProbeTemplate" }} +httpGet: + path: /metrics + port: {{ tuple "oslo_cache" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +{{- end }} + +{{- if .Values.manifests.statefulset }} +{{- $envAll := . }} + +{{- $rcControllerName := printf "%s-%s" $envAll.deployment_name "memcached" }} +{{- $configMapBinName := printf "%s-%s" $envAll.deployment_name "memcached-bin" }} + +{{ tuple $envAll "memcached" $rcControllerName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ $rcControllerName | quote }} + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "memcached" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + podManagementPolicy: Parallel + replicas: {{ .Values.pod.replicas.server }} + serviceName: "{{ tuple "oslo_cache" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }}" + selector: + matchLabels: +{{ tuple $envAll "memcached" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} + template: + metadata: + annotations: +{{ dict "envAll" $envAll "podName" "memcached" "containerNames" (list "init" "memcached") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + labels: +{{ tuple $envAll "memcached" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "server" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + shareProcessNamespace: true + serviceAccountName: {{ $rcControllerName | quote }} + affinity: +{{ tuple $envAll "memcached" "server" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} + nodeSelector: + {{ .Values.labels.server.node_selector_key }}: {{ .Values.labels.server.node_selector_value | quote }} +{{ if $envAll.Values.pod.tolerations.memcached.enabled }} +{{ tuple $envAll "memcached" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }} +{{ end }} + terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.memcached.timeout | default "30" }} + initContainers: +{{ tuple $envAll "memcached" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} +{{ dict "envAll" $envAll | include "helm-toolkit.snippets.kubernetes_apparmor_loader_init_container" | indent 8 }} + containers: + - name: memcached +{{ tuple $envAll "memcached" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "server" "container" "memcached" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + env: + - name: MEMCACHED_PORT + value: {{ tuple "oslo_cache" "internal" "memcache" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: MEMCACHED_MAX_CONNECTIONS + value: {{ .Values.conf.memcached.max_connections | quote }} + - name: MEMCACHED_MEMORY + value: {{ .Values.conf.memcached.memory | quote }} + command: + - /tmp/memcached.sh + ports: + - containerPort: {{ tuple "oslo_cache" "internal" "memcache" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +{{ dict "envAll" $envAll "component" "memcached" "container" "memcached" "type" "readiness" "probeTemplate" (include "memcachedProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }} +{{ dict "envAll" $envAll "component" "memcached" "container" "memcached" "type" "liveness" "probeTemplate" (include "memcachedProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }} + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: memcached-bin + mountPath: /tmp/memcached.sh + subPath: memcached.sh + readOnly: true +{{- if .Values.monitoring.prometheus.enabled }} + - name: memcached-exporter + env: + - name: MEMCACHED_HOST + value: 127.0.0.1 + - name: MEMCACHED_PORT + value: {{ tuple "oslo_cache" "internal" "memcache" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} +{{ tuple $envAll "prometheus_memcached_exporter" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.prometheus_memcached_exporter | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "server" "container" "memcached_exporter" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/memcached-exporter.sh + - start + ports: + - name: metrics + containerPort: {{ tuple "oslo_cache" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +{{ dict "envAll" $envAll "component" "memcached" "container" "memcached_exporter" "type" "readiness" "probeTemplate" (include "exporterProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }} +{{ dict "envAll" $envAll "component" "memcached" "container" "memcached_exporter" "type" "liveness" "probeTemplate" (include "exporterProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }} + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: memcached-bin + mountPath: /tmp/memcached-exporter.sh + subPath: memcached-exporter.sh + readOnly: true +{{- end }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: memcached-bin + configMap: + name: {{ $configMapBinName | quote }} + defaultMode: 360 +{{ dict "envAll" $envAll "component" "memcached" "requireSys" true | include "helm-toolkit.snippets.kubernetes_apparmor_volumes" | indent 8 }} +{{- end }} diff --git a/memcached/values.yaml b/memcached/values.yaml new file mode 100644 index 0000000000..41fcb50865 --- /dev/null +++ b/memcached/values.yaml @@ -0,0 +1,250 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for memcached. +# This is a YAML-formatted file. +# Declare name/value pairs to be passed into your templates. +# name: value + +--- +conf: + memcached: + max_connections: 8192 + # NOTE(pordirect): this should match the value in + # `pod.resources.memcached.memory` + memory: 1024 + stats_cachedump: + enabled: true + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - memcached-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + memcached: + jobs: null + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + +secrets: + oci_image_registry: + memcached: memcached-oci-image-registry-key + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + memcached: + username: memcached + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + oslo_cache: + namespace: null + host_fqdn_override: + default: null + hosts: + default: memcached + port: + memcache: + default: 11211 + metrics: + default: 9150 + kube_dns: + namespace: kube-system + name: kubernetes-dns + hosts: + default: kube-dns + host_fqdn_override: + default: null + path: + default: null + scheme: http + port: + dns_tcp: + default: 53 + dns: + default: 53 + protocol: UDP + +network: + memcached: {} + +network_policy: + memcached: + ingress: + - {} + egress: + - {} + +monitoring: + prometheus: + enabled: false + memcached_exporter: + scrape: true + +images: + pull_policy: IfNotPresent + tags: + dep_check: 'quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal' + memcached: 'docker.io/library/memcached:1.5.5' + prometheus_memcached_exporter: docker.io/prom/memcached-exporter:v0.4.1 + image_repo_sync: docker.io/library/docker:17.07.0 + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +labels: + server: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +manifests: + configmap_bin: true + statefulset: true + job_image_repo_sync: true + network_policy: false + service: true + secret_registry: true + +pod: + security_context: + server: + pod: + runAsUser: 65534 + runAsNonRoot: true + fsGroup: 65534 + container: + memcached: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + memcached_exporter: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL + probes: + memcached: + memcached: + readiness: + enabled: True + params: + initialDelaySeconds: 0 + periodSeconds: 10 + timeoutSeconds: 5 + liveness: + enabled: True + params: + initialDelaySeconds: 10 + periodSeconds: 15 + timeoutSeconds: 10 + memcached_exporter: + liveness: + enabled: True + params: + initialDelaySeconds: 15 + periodSeconds: 60 + timeoutSeconds: 10 + readiness: + enabled: True + params: + initialDelaySeconds: 5 + periodSeconds: 60 + timeoutSeconds: 10 + affinity: + anti: + topologyKey: + default: kubernetes.io/hostname + type: + default: requiredDuringSchedulingIgnoredDuringExecution + weight: + default: 10 + tolerations: + memcached: + enabled: false + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + lifecycle: + upgrades: + deployments: + pod_replacement_strategy: RollingUpdate + revision_history: 3 + rolling_update: + max_unavailable: 1 + termination_grace_period: + memcached: + timeout: 30 + replicas: + server: 1 + resources: + enabled: false + memcached: + limits: + cpu: "2000m" + memory: "1024Mi" + requests: + cpu: "500m" + memory: "128Mi" + prometheus_memcached_exporter: + limits: + memory: "1024Mi" + cpu: "2000m" + requests: + cpu: 500m + memory: 128Mi + jobs: + image_repo_sync: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" +... diff --git a/metacontroller/Chart.yaml b/metacontroller/Chart.yaml new file mode 100644 index 0000000000..e57f5f8c95 --- /dev/null +++ b/metacontroller/Chart.yaml @@ -0,0 +1,31 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: v0.4.2 +description: A Helm chart for Metacontroller +name: metacontroller +version: 2024.2.0 +home: https://metacontroller.app/ +keywords: + - CRDs + - metacontroller +sources: + - https://github.com/GoogleCloudPlatform/metacontroller +maintainers: + - name: OpenStack-Helm Authors +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit + version: ">= 0.1.0" +... diff --git a/metacontroller/templates/crds.yaml b/metacontroller/templates/crds.yaml new file mode 100644 index 0000000000..0355dbc7d4 --- /dev/null +++ b/metacontroller/templates/crds.yaml @@ -0,0 +1,333 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.crds }} +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: compositecontrollers.metacontroller.k8s.io + annotations: + "api-approved.kubernetes.io": "https://github.com/kubernetes/kubernetes/pull/78458" +spec: + group: metacontroller.k8s.io + versions: + - name: v1alpha1 + served: true + storage: true + subresources: + status: {} + schema: + openAPIV3Schema: + type: object + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + type: object + properties: + generateSelector: + type: boolean + resyncPeriodSeconds: + format: int32 + type: integer + parentResource: + properties: + apiVersion: + description: APIVersion is the combination of group & version of + the resource + type: string + resource: + description: Resource is the name of the resource. Its also the + plural of Kind + type: string + revisionHistory: + properties: + fieldPaths: + items: + type: string + type: array + type: object + required: + - apiVersion + - resource + type: object + childResources: + items: + properties: + apiVersion: + description: APIVersion is the combination of group & version + of the resource + type: string + resource: + description: Resource is the name of the resource. Its also the + plural of Kind + type: string + updateStrategy: + properties: + method: + description: ChildUpdateMethod represents a typed constant + to determine the update strategy of a child resource + type: string + statusChecks: + properties: + conditions: + items: + properties: + reason: + type: string + status: + type: string + type: + type: string + required: + - type + type: object + type: array + type: object + type: object + required: + - apiVersion + - resource + type: object + type: array + hooks: + properties: + finalize: + description: Hook refers to the logic that builds the desired state + of resources + properties: + inline: + description: Inline invocation to arrive at desired state + properties: + funcName: + type: string + type: object + webhook: + description: Webhook invocation to arrive at desired state + properties: + path: + type: string + service: + properties: + name: + type: string + namespace: + type: string + port: + format: int32 + type: integer + protocol: + type: string + required: + - name + - namespace + type: object + timeout: + type: string + url: + type: string + type: object + type: object + postUpdateChild: + description: Hook refers to the logic that builds the desired state + of resources + properties: + inline: + description: Inline invocation to arrive at desired state + properties: + funcName: + type: string + type: object + webhook: + description: Webhook invocation to arrive at desired state + properties: + path: + type: string + service: + properties: + name: + type: string + namespace: + type: string + port: + format: int32 + type: integer + protocol: + type: string + required: + - name + - namespace + type: object + timeout: + type: string + url: + type: string + type: object + type: object + preUpdateChild: + description: Hook refers to the logic that builds the desired state + of resources + properties: + inline: + description: Inline invocation to arrive at desired state + properties: + funcName: + type: string + type: object + webhook: + description: Webhook invocation to arrive at desired state + properties: + path: + type: string + service: + properties: + name: + type: string + namespace: + type: string + port: + format: int32 + type: integer + protocol: + type: string + required: + - name + - namespace + type: object + timeout: + type: string + url: + type: string + type: object + type: object + sync: + description: Hook refers to the logic that builds the desired state + of resources + properties: + inline: + description: Inline invocation to arrive at desired state + properties: + funcName: + type: string + type: object + webhook: + description: Webhook invocation to arrive at desired state + properties: + path: + type: string + service: + properties: + name: + type: string + namespace: + type: string + port: + format: int32 + type: integer + protocol: + type: string + required: + - name + - namespace + type: object + timeout: + type: string + url: + type: string + type: object + type: object + type: object + required: + - parentResource + status: + type: object + required: + - metadata + - spec + scope: Cluster + names: + plural: compositecontrollers + singular: compositecontroller + kind: CompositeController + shortNames: + - cc + - cctl +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: decoratorcontrollers.metacontroller.k8s.io + annotations: + "api-approved.kubernetes.io": "https://github.com/kubernetes/kubernetes/pull/78458" +spec: + group: metacontroller.k8s.io + versions: + - name: v1alpha1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + scope: Cluster + names: + plural: decoratorcontrollers + singular: decoratorcontroller + kind: DecoratorController + shortNames: + - dec + - decorators +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: controllerrevisions.metacontroller.k8s.io + annotations: + "api-approved.kubernetes.io": "https://github.com/kubernetes/kubernetes/pull/78458" +spec: + group: metacontroller.k8s.io + versions: + - name: v1alpha1 + served: true + storage: true + schema: + openAPIV3Schema: + type: object + properties: + spec: + type: object + scope: Namespaced + names: + plural: controllerrevisions + singular: controllerrevision + kind: ControllerRevision +{{- end }} diff --git a/metacontroller/templates/job-image-repo-sync.yaml b/metacontroller/templates/job-image-repo-sync.yaml new file mode 100644 index 0000000000..7cc55d2f6c --- /dev/null +++ b/metacontroller/templates/job-image-repo-sync.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "metacontroller" -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} \ No newline at end of file diff --git a/metacontroller/templates/secret-registry.yaml b/metacontroller/templates/secret-registry.yaml new file mode 100644 index 0000000000..da979b3223 --- /dev/null +++ b/metacontroller/templates/secret-registry.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_registry .Values.endpoints.oci_image_registry.auth.enabled }} +{{ include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) }} +{{- end }} diff --git a/metacontroller/templates/service.yaml b/metacontroller/templates/service.yaml new file mode 100644 index 0000000000..62674a661b --- /dev/null +++ b/metacontroller/templates/service.yaml @@ -0,0 +1,32 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.service }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "metacontroller" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + namespace: {{ .Release.Namespace }} + labels: +{{ tuple $envAll "metacontroller" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + clusterIP: None + ports: + - name: metacontroller + port: {{ tuple "metacontroller" "internal" "metacontroller" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + selector: +{{ tuple $envAll "metacontroller" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- end }} \ No newline at end of file diff --git a/metacontroller/templates/statefulset.yaml b/metacontroller/templates/statefulset.yaml new file mode 100644 index 0000000000..2472ec4760 --- /dev/null +++ b/metacontroller/templates/statefulset.yaml @@ -0,0 +1,94 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.statefulset }} +{{- $envAll := . }} + +{{- $serviceAccountName := "metacontroller-serviceaccount" }} +{{ tuple $envAll "metacontroller" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} + +{{ $controllerName := printf "%s-%s" .Release.Namespace $serviceAccountName }} +--- +{{- if .Values.manifests.rbac }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ $controllerName }} +rules: +- apiGroups: + - "*" + resources: + - "*" + verbs: + - "*" +{{- end }} +--- +{{- if .Values.manifests.rbac }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ $controllerName }} +subjects: +- kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ $controllerName }} + apiGroup: rbac.authorization.k8s.io +{{- end }} +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: metacontroller + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 4 }} + namespace: {{ .Release.Namespace }} + labels: +{{ tuple $envAll "metacontroller" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + selector: + matchLabels: +{{ tuple $envAll "metacontroller" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} + serviceName: {{ tuple "metacontroller" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + podManagementPolicy: "Parallel" + replicas: {{ .Values.pod.replicas.metacontroller }} + template: + metadata: + labels: +{{ tuple $envAll "metacontroller" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ dict "envAll" $envAll "podName" "metacontroller" "containerNames" (list "metacontroller") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" . "application" "metacontroller" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.server.timeout | default "30" }} + nodeSelector: + {{ .Values.labels.server.node_selector_key }}: {{ .Values.labels.server.node_selector_value | quote }} + containers: + - name: metacontroller +{{ tuple $envAll "metacontroller" | include "helm-toolkit.snippets.image" | indent 8 }} +{{ tuple $envAll $envAll.Values.pod.resources.metacontroller | include "helm-toolkit.snippets.kubernetes_resources" | indent 8 }} +{{ dict "envAll" $envAll "application" "metacontroller" "container" "metacontroller" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 8 }} + ports: + - name: metacontroller + containerPort: {{ tuple "metacontroller" "internal" "metacontroller" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + command: + - /usr/bin/metacontroller + args: + - --logtostderr + - -v=6 + - --discovery-interval=20s +{{- end }} diff --git a/metacontroller/values.yaml b/metacontroller/values.yaml new file mode 100644 index 0000000000..22b64bd4d6 --- /dev/null +++ b/metacontroller/values.yaml @@ -0,0 +1,135 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for elasticsearch +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +--- +release_group: null + +images: + tags: + metacontroller: metacontrollerio/metacontroller:v0.4.2 + dep_check: quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal + image_repo_sync: docker.io/library/docker:17.07.0 + pull_policy: IfNotPresent + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +labels: + server: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - metacontroller-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry +pod: + lifecycle: + termination_grace_period: + server: + timeout: 600 + resources: + enabled: false + metacontroller: + limits: + memory: "1024Mi" + cpu: "2000m" + requests: + memory: "128Mi" + cpu: "500m" + replicas: + metacontroller: 1 + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + weight: + default: 10 + security_context: + metacontroller: + pod: + runAsUser: 34356 + container: + metacontroller: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + +secrets: + oci_image_registry: + metacontroller: metacontroller-oci-image-registry-key + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + metacontroller: + username: metacontroller + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + metacontroller: + hosts: + default: metacontroller + host_fqdn_override: + default: null + port: + metacontroller: + default: 8083 + +manifests: + secret_registry: true + service: true + statefulset: true + job_image_repo_sync: true + crds: true + rbac: true + + +... diff --git a/mongodb/Chart.yaml b/mongodb/Chart.yaml new file mode 100644 index 0000000000..caac77f1b4 --- /dev/null +++ b/mongodb/Chart.yaml @@ -0,0 +1,29 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: v3.4.9 +description: OpenStack-Helm MongoDB +name: mongodb +version: 2024.2.0 +home: https://www.mongodb.com +sources: + - https://github.com/mongodb/mongo + - https://opendev.org/openstack/openstack-helm +maintainers: + - name: OpenStack-Helm Authors +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit + version: ">= 0.1.0" +... diff --git a/mongodb/templates/bin/_start.sh.tpl b/mongodb/templates/bin/_start.sh.tpl new file mode 100644 index 0000000000..08a77b505c --- /dev/null +++ b/mongodb/templates/bin/_start.sh.tpl @@ -0,0 +1,45 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +mongod --config /etc/mongodb.conf --auth & + +t=0 +until mongo --eval "db.adminCommand('ping')"; do + echo "waiting for mongodb to start" + sleep 1 + t=$(($t+1)) + if [ $t -ge 30 ] ; then + echo "mongodb did not start, giving up" + exit 1 + fi +done + +#NOTE(portdirect): stop sending commands to stdout to prevent root password +# being sent to logs. +set +x +mongo admin \ + --username "${ADMIN_USER}" \ + --password "${ADMIN_PASS}" \ + --eval "db.changeUserPassword(\"${ADMIN_USER}\", \"${ADMIN_PASS}\")" || \ + mongo admin \ + --eval "db.createUser({ user: \"${ADMIN_USER}\", \ + pwd: \"${ADMIN_PASS}\", \ + roles: [ { role: \"userAdminAnyDatabase\", \ + db: \"admin\" } ] });" +set -x +wait diff --git a/mongodb/templates/configmap-bin.yaml b/mongodb/templates/configmap-bin.yaml new file mode 100644 index 0000000000..47e7302cb6 --- /dev/null +++ b/mongodb/templates/configmap-bin.yaml @@ -0,0 +1,29 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_bin }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: mongodb-bin +data: +{{- if .Values.images.local_registry.active }} + image-repo-sync.sh: | +{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} +{{- end }} + start.sh: | +{{ tuple "bin/_start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} diff --git a/mongodb/templates/configmap-etc.yaml b/mongodb/templates/configmap-etc.yaml new file mode 100644 index 0000000000..f2cbbf24cb --- /dev/null +++ b/mongodb/templates/configmap-etc.yaml @@ -0,0 +1,26 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_etc }} +{{- $envAll := . }} + +--- +apiVersion: v1 +kind: Secret +metadata: + name: mongodb-etc +type: Opaque +data: + mongodb.conf: {{ tuple "secrets/_mongodb.cnf.tpl" . | include "helm-toolkit.utils.template" | b64enc }} +{{- end }} \ No newline at end of file diff --git a/mongodb/templates/job-image-repo-sync.yaml b/mongodb/templates/job-image-repo-sync.yaml new file mode 100644 index 0000000000..a732bef2e2 --- /dev/null +++ b/mongodb/templates/job-image-repo-sync.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "mongodb" -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/mongodb/templates/secret-db-root-password.yaml b/mongodb/templates/secret-db-root-password.yaml new file mode 100644 index 0000000000..5ad7072626 --- /dev/null +++ b/mongodb/templates/secret-db-root-password.yaml @@ -0,0 +1,26 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_db_root_creds }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: mongodb-root-creds +type: Opaque +data: + MONGODB_ROOT_PASSWORD: {{ .Values.endpoints.mongodb.auth.admin.password | b64enc }} + MONGODB_ROOT_USERNAME: {{ .Values.endpoints.mongodb.auth.admin.username | b64enc }} +{{- end }} diff --git a/mongodb/templates/secret-registry.yaml b/mongodb/templates/secret-registry.yaml new file mode 100644 index 0000000000..da979b3223 --- /dev/null +++ b/mongodb/templates/secret-registry.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_registry .Values.endpoints.oci_image_registry.auth.enabled }} +{{ include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) }} +{{- end }} diff --git a/mongodb/templates/secrets/_mongodb.cnf.tpl b/mongodb/templates/secrets/_mongodb.cnf.tpl new file mode 100644 index 0000000000..9180c2dfa9 --- /dev/null +++ b/mongodb/templates/secrets/_mongodb.cnf.tpl @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +bind_ip = {{ .Values.endpoints.mongodb.bind_ip}} +port = {{ .Values.endpoints.mongodb.port.mongodb.default}} +# Where to store the data. +dbpath=/var/lib/mongodb diff --git a/mongodb/templates/service.yaml b/mongodb/templates/service.yaml new file mode 100644 index 0000000000..5b2a9d54b5 --- /dev/null +++ b/mongodb/templates/service.yaml @@ -0,0 +1,28 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.service }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "mongodb" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +spec: + ports: + - name: db + port: {{ tuple "mongodb" "internal" "mongodb" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + selector: +{{ tuple $envAll "mongodb" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- end }} diff --git a/mongodb/templates/statefulset.yaml b/mongodb/templates/statefulset.yaml new file mode 100644 index 0000000000..7456a0778d --- /dev/null +++ b/mongodb/templates/statefulset.yaml @@ -0,0 +1,146 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.statefulset }} +{{- $envAll := . }} + +{{- $serviceAccountName := "mongodb" }} +{{ tuple $envAll "mongodb" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: mongodb + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "mongodb" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + serviceName: {{ tuple "mongodb" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + replicas: {{ .Values.pod.replicas.server }} + selector: + matchLabels: +{{ tuple $envAll "mongodb" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} + template: + metadata: + labels: +{{ tuple $envAll "mongodb" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + spec: + serviceAccountName: {{ $serviceAccountName }} + affinity: +{{ tuple $envAll "mongodb" "server" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} + nodeSelector: + {{ .Values.labels.server.node_selector_key }}: {{ .Values.labels.server.node_selector_value }} + initContainers: +{{ tuple $envAll "mongodb" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} +{{- if $envAll.Values.volume.chown_on_start }} + - name: mongodb-perms +{{ tuple $envAll "mongodb" | include "helm-toolkit.snippets.image" | indent 10 }} + securityContext: + runAsUser: 0 +{{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + command: + - chown + - -R + - "mongodb:" + - {{ $envAll.Values.volume.host.host_path }} + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: mongodb-data + mountPath: {{ $envAll.Values.volume.host.host_path }} +{{- end }} + containers: + - name: mongodb +{{ tuple $envAll "mongodb" | include "helm-toolkit.snippets.image" | indent 10 }} + ports: + - containerPort: {{ tuple "mongodb" "internal" "mongodb" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + env: + - name: ADMIN_PASS + valueFrom: + secretKeyRef: + name: mongodb-root-creds + key: MONGODB_ROOT_PASSWORD + - name: ADMIN_USER + valueFrom: + secretKeyRef: + name: mongodb-root-creds + key: MONGODB_ROOT_USERNAME + command: + - /tmp/start.sh + livenessProbe: + exec: + command: + - mongo + - --eval + - "db.adminCommand('ping')" + initialDelaySeconds: 20 + timeoutSeconds: 5 + readinessProbe: + exec: + command: + - mongo + - --eval + - "db.adminCommand('ping')" + initialDelaySeconds: 20 + timeoutSeconds: 5 +{{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: mongodb-bin + mountPath: /tmp/start.sh + subPath: start.sh + readOnly: true + - name: mongodb-bin + mountPath: /tmp/setup_admin_user.sh + subPath: setup_admin_user.sh + readOnly: true + - name: mongodb-etc + mountPath: /etc/mongodb.conf + subPath: mongodb.conf + readOnly: true + - name: mongodb-data + mountPath: /data/db + volumes: + - name: pod-tmp + emptyDir: {} + - name: mongodb-bin + configMap: + name: mongodb-bin + defaultMode: 0555 + - name: mongodb-etc + secret: + secretName: mongodb-etc + defaultMode: 0444 +{{- if not .Values.volume.enabled }} + - name: mongodb-data + hostPath: + path: {{ .Values.volume.host_path }} +{{- else }} + volumeClaimTemplates: + - metadata: + name: mongodb-data + annotations: + {{ .Values.volume.class_path }}: {{ .Values.volume.class_name }} + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: {{ .Values.volume.size }} +{{- end }} +{{- end }} diff --git a/mongodb/values.yaml b/mongodb/values.yaml new file mode 100644 index 0000000000..e0d353e5a8 --- /dev/null +++ b/mongodb/values.yaml @@ -0,0 +1,151 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for mongodb. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +--- +release_group: null + +pod: + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + weight: + default: 10 + replicas: + # only 1 replica currently supported + server: 1 + resources: + enabled: false + server: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + jobs: + image_repo_sync: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + +# using dockerhub mongodb: https://hub.docker.com/r/library/mongo/tags/ +images: + tags: + mongodb: docker.io/library/mongo:3.4.9-jessie + dep_check: quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal + image_repo_sync: docker.io/library/docker:17.07.0 + pull_policy: "IfNotPresent" + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +volume: + chown_on_start: true + enabled: true + size: 5Gi + class_name: general + class_path: volume.beta.kubernetes.io/storage-class + host: + host_path: /var/lib/openstack-helm/mongodb + +labels: + server: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +secrets: + oci_image_registry: + mongodb: mongodb-oci-image-registry-key + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + mongodb: + username: mongodb + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + mongodb: + auth: + admin: + username: root + password: password + hosts: + default: mongodb + host_fqdn_override: + default: null + path: null + scheme: mongodb + bind_ip: 0.0.0.0 + port: + mongodb: + default: 27017 + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - mongodb-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + mongodb: + jobs: null + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + +manifests: + configmap_bin: true + configmap_etc: true + job_image_repo_sync: true + secret_db_root_creds: true + secret_registry: true + service: true + statefulset: true +... diff --git a/nagios/Chart.yaml b/nagios/Chart.yaml new file mode 100644 index 0000000000..97a155f9f8 --- /dev/null +++ b/nagios/Chart.yaml @@ -0,0 +1,28 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: v1.0.0 +description: OpenStack-Helm Nagios +name: nagios +version: 2024.2.0 +home: https://www.nagios.org +sources: + - https://opendev.org/openstack/openstack-helm-addons +maintainers: + - name: OpenStack-Helm Authors +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit + version: ">= 0.1.0" +... diff --git a/nagios/templates/bin/_apache.sh.tpl b/nagios/templates/bin/_apache.sh.tpl new file mode 100644 index 0000000000..a7b98d1931 --- /dev/null +++ b/nagios/templates/bin/_apache.sh.tpl @@ -0,0 +1,44 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ev + +COMMAND="${@:-start}" + +function start () { + + if [ -f /etc/apache2/envvars ]; then + # Loading Apache2 ENV variables + source /etc/httpd/apache2/envvars + fi + # Apache gets grumpy about PID files pre-existing + rm -f /etc/httpd/logs/httpd.pid + + if [ -f /usr/local/apache2/conf/.htpasswd ]; then + htpasswd -b /usr/local/apache2/conf/.htpasswd "$NAGIOSADMIN_USER" "$NAGIOSADMIN_PASS" + else + htpasswd -cb /usr/local/apache2/conf/.htpasswd "$NAGIOSADMIN_USER" "$NAGIOSADMIN_PASS" + fi + + #Launch Apache on Foreground + exec httpd -DFOREGROUND +} + +function stop () { + apachectl -k graceful-stop +} + +$COMMAND diff --git a/nagios/templates/bin/_nagios-readiness.sh.tpl b/nagios/templates/bin/_nagios-readiness.sh.tpl new file mode 100644 index 0000000000..e45618aa03 --- /dev/null +++ b/nagios/templates/bin/_nagios-readiness.sh.tpl @@ -0,0 +1,25 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# NOTE(sw5822): Redirect no-op operator output to Nagios log file to clean out +# Nagios's log file, since Nagios doesn't support logging to /dev/null +: > /opt/nagios/var/log/nagios.log + +# Check whether Nagios endpoint is reachable +reply=$(curl -s -o /dev/null -w %{http_code} http://127.0.0.1:8000/nagios) +if [ \"$reply\" -lt 200 -o \"$reply\" -ge 400 ]; then + exit 1 +fi diff --git a/nagios/templates/bin/_selenium-tests.py.tpl b/nagios/templates/bin/_selenium-tests.py.tpl new file mode 100644 index 0000000000..9a8dd1d179 --- /dev/null +++ b/nagios/templates/bin/_selenium-tests.py.tpl @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +import os +import logging +import sys +from selenium import webdriver +from selenium.webdriver.common.by import By +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support import expected_conditions as EC +from selenium.webdriver.chrome.options import Options +{{- if .Values.selenium_v4 }} +from selenium.webdriver.chrome.service import Service +{{- end }} +from selenium.common.exceptions import TimeoutException +from selenium.common.exceptions import NoSuchElementException +from selenium.common.exceptions import ScreenshotException + +# Create logger, console handler and formatter +logger = logging.getLogger('Nagios Selenium Tests') +logger.setLevel(logging.DEBUG) +ch = logging.StreamHandler() +ch.setLevel(logging.DEBUG) +formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s' +) + +# Set the formatter and add the handler +ch.setFormatter(formatter) +logger.addHandler(ch) + +def get_variable(env_var): + if env_var in os.environ: + logger.info('Found "{}"'.format(env_var)) + return os.environ[env_var] + else: + logger.critical('Variable "{}" is not defined!'.format(env_var)) + sys.exit(1) + +def click_link_by_name(link_name): + try: + logger.info("Clicking '{}' link".format(link_name)) +{{- if .Values.selenium_v4 }} + link = browser.find_element(By.LINK_TEXT, link_name) +{{- else }} + link = browser.find_element_by_link_text(link_name) +{{- end }} + link.click() + except NoSuchElementException: + logger.error("Failed clicking '{}' link".format(link_name)) + browser.quit() + sys.exit(1) + +def take_screenshot(page_name, artifacts_dir='/tmp/artifacts/'): # nosec + file_name = page_name.replace(' ', '_') + try: + el = WebDriverWait(browser, 15) + browser.save_screenshot('{}{}.png'.format(artifacts_dir, file_name)) + logger.info("Successfully captured {} screenshot".format(page_name)) + except ScreenshotException: + logger.error("Failed to capture {} screenshot".format(page_name)) + browser.quit() + sys.exit(1) + +username = get_variable('NAGIOS_USER') +password = get_variable('NAGIOS_PASSWORD') +nagios_uri = get_variable('NAGIOS_URI') +nagios_url = 'http://{0}:{1}@{2}'.format(username, password, nagios_uri) + +chrome_driver = '/etc/selenium/chromedriver' +options = Options() +options.add_argument('--headless=new') +options.add_argument('--no-sandbox') +options.add_argument('--window-size=1920x1080') + +{{- if .Values.selenium_v4 }} +service = Service(executable_path=chrome_driver) +browser = webdriver.Chrome(service=service, options=options) +{{- else }} +browser = webdriver.Chrome(chrome_driver, chrome_options=options) +{{- end }} + +try: + logger.info('Attempting to connect to Nagios') + browser.get(nagios_url) + el = WebDriverWait(browser, 15).until( + EC.title_contains('Nagios') + ) + logger.info('Connected to Nagios') +except TimeoutException: + logger.critical('Timed out waiting for Nagios') + browser.quit() + sys.exit(1) + +try: + logger.info('Switching Focus to Navigation side frame') + sideFrame = browser.switch_to.frame('side') +except NoSuchElementException: + logger.error('Failed selecting side frame') + browser.quit() + sys.exit(1) + +try: + logger.info('Attempting to visit Services page') + click_link_by_name('Services') + take_screenshot('Nagios Services') +except TimeoutException: + logger.error('Failed to load Services page') + browser.quit() + sys.exit(1) + +try: + logger.info('Attempting to visit Host Groups page') + click_link_by_name('Host Groups') + take_screenshot('Nagios Host Groups') +except TimeoutException: + logger.error('Failed to load Host Groups page') + browser.quit() + sys.exit(1) + +try: + logger.info('Attempting to visit Hosts page') + click_link_by_name('Hosts') + take_screenshot('Nagios Hosts') +except TimeoutException: + logger.error('Failed to load Hosts page') + browser.quit() + sys.exit(1) + +logger.info("The following screenshots were captured:") +for root, dirs, files in os.walk("/tmp/artifacts/"): # nosec + for name in files: + logger.info(os.path.join(root, name)) + +browser.quit() diff --git a/nagios/templates/configmap-additional-plugins.yaml b/nagios/templates/configmap-additional-plugins.yaml new file mode 100644 index 0000000000..42002062a8 --- /dev/null +++ b/nagios/templates/configmap-additional-plugins.yaml @@ -0,0 +1,27 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_additional_plugins }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: nagios-additional-plugins +type: Opaque +data: +{{- range .Values.conf.nagios.additionalPlugins }} + {{ .name }}: {{ .content | b64enc | quote }} +{{- end }} +{{- end }} diff --git a/nagios/templates/configmap-bin.yaml b/nagios/templates/configmap-bin.yaml new file mode 100644 index 0000000000..c46b145cc4 --- /dev/null +++ b/nagios/templates/configmap-bin.yaml @@ -0,0 +1,31 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_bin }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: nagios-bin +data: + apache.sh: | +{{ tuple "bin/_apache.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + nagios-readiness.sh: | +{{ tuple "bin/_nagios-readiness.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + selenium-tests.py: | +{{ tuple "bin/_selenium-tests.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + image-repo-sync.sh: |+ +{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} +{{- end }} diff --git a/nagios/templates/configmap-etc.yaml b/nagios/templates/configmap-etc.yaml new file mode 100644 index 0000000000..28ef873d93 --- /dev/null +++ b/nagios/templates/configmap-etc.yaml @@ -0,0 +1,35 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_etc }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: nagios-etc +type: Opaque +data: + {{- if not (empty .Values.conf.nagios.query_es_clauses) }} + query_es_clauses.json: {{ .Values.conf.nagios.query_es_clauses | toJson | b64enc }} + {{- end }} +{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.nagios.nagios.template "key" "nagios.cfg" "format" "Secret") | indent 2 }} +{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.nagios.cgi.template "key" "cgi.cfg" "format" "Secret") | indent 2 }} +{{- range $objectType, $config := $envAll.Values.conf.nagios.objects }} +{{- $objectFile := printf "%s.cfg" $objectType -}} +{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" $config.template "key" $objectFile "format" "Secret") | indent 2 }} +{{- end }} + # NOTE(portdirect): this must be last, to work round helm ~2.7 bug. +{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.httpd "key" "httpd.conf" "format" "Secret") | indent 2 }} +{{- end }} diff --git a/nagios/templates/deployment.yaml b/nagios/templates/deployment.yaml new file mode 100644 index 0000000000..77c64fa83a --- /dev/null +++ b/nagios/templates/deployment.yaml @@ -0,0 +1,271 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "apacheProxyReadinessProbeTemplate" }} +tcpSocket: + port: {{ tuple "nagios" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +{{- end }} + +{{- define "nagiosReadinessProbeTemplate" }} +exec: + command: + - /tmp/nagios-readiness.sh +{{- end }} + +{{- if .Values.manifests.deployment }} +{{- $envAll := . }} + +{{- $nagiosUserSecret := .Values.secrets.nagios.admin }} + +{{- $serviceAccountName := "nagios" }} +{{ tuple $envAll "nagios" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ $serviceAccountName }} +rules: + - apiGroups: + - "" + resources: + - nodes + - nodes/proxy + - services + - endpoints + - pods + - pods/exec + - persistentvolumes + - persistentvolumeclaims + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ $serviceAccountName }} + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nagios + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "nagios" "monitoring" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + replicas: {{ .Values.pod.replicas.nagios }} + selector: + matchLabels: +{{ tuple $envAll "nagios" "monitoring" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} +{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "nagios" "monitoring" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} +{{ dict "envAll" $envAll "podName" "nagios" "containerNames" (list "apache-proxy" "nagios" "init" "define-nagios-hosts") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "monitoring" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + affinity: +{{ tuple $envAll "nagios" "monitoring" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} + nodeSelector: + {{ .Values.labels.nagios.node_selector_key }}: {{ .Values.labels.nagios.node_selector_value | quote }} + terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.nagios.timeout | default "30" }} + {{- if or ( gt .Capabilities.KubeVersion.Major "1" ) ( ge .Capabilities.KubeVersion.Minor "10" ) }} + shareProcessNamespace: true + {{- else }} + hostPID: true + {{- end }} + initContainers: +{{ tuple $envAll "nagios" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + - name: define-nagios-hosts +{{ tuple $envAll "nagios" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.nagios | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "monitoring" "container" "define_nagios_hosts" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /usr/lib/nagios/plugins/define-nagios-hosts.py + - --object_file_loc + - /opt/nagios/etc/conf.d/nagios-hosts.cfg + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: nagios-confd + mountPath: /opt/nagios/etc/conf.d + env: +{{- if .Values.pod.env }} +{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env | indent 12 }} +{{- end }} + containers: + - name: apache-proxy +{{ tuple $envAll "apache_proxy" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.apache_proxy | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "monitoring" "container" "apache_proxy" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} +{{ dict "envAll" $envAll "component" "monitoring" "container" "apache_proxy" "type" "readiness" "probeTemplate" (include "apacheProxyReadinessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }} + command: + - /tmp/apache.sh + - start + ports: + - name: http + containerPort: {{ tuple "nagios" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + env: + - name: NAGIOSADMIN_USER + valueFrom: + secretKeyRef: + name: {{ $nagiosUserSecret }} + key: NAGIOSADMIN_USER + - name: NAGIOSADMIN_PASS + valueFrom: + secretKeyRef: + name: {{ $nagiosUserSecret }} + key: NAGIOSADMIN_PASS + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: nagios-bin + mountPath: /tmp/apache.sh + subPath: apache.sh + readOnly: true + - name: nagios-etc + mountPath: /usr/local/apache2/conf/httpd.conf + subPath: httpd.conf + readOnly: true + - name: nagios +{{ tuple $envAll "nagios" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.nagios | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "monitoring" "container" "nagios" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} +{{ dict "envAll" $envAll "component" "monitoring" "container" "nagios" "type" "readiness" "probeTemplate" (include "nagiosReadinessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }} + ports: + - name: nagios + containerPort: {{ tuple "nagios" "internal" "nagios" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + env: +{{- if .Values.pod.env }} +{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env | indent 12 }} +{{- end }} + - name: SNMP_NOTIF_PRIMARY_TARGET_WITH_PORT + value: {{ $envAll.Values.conf.nagios.notification.snmp.primary_target }} + - name: SNMP_NOTIF_SECONDARY_TARGET_WITH_PORT + value: {{ $envAll.Values.conf.nagios.notification.snmp.secondary_target }} + - name: REST_NOTIF_PRIMARY_TARGET_URL + value: {{ $envAll.Values.conf.nagios.notification.http.primary_target }} + - name: REST_NOTIF_SECONDARY_TARGET_URL + value: {{ $envAll.Values.conf.nagios.notification.http.secondary_target }} + - name: CEPH_MGR_SERVICE + value: {{ tuple "ceph_mgr" "internal" "metrics" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}/metrics + - name: PROMETHEUS_SERVICE + valueFrom: + secretKeyRef: + name: {{ $nagiosUserSecret }} + key: PROMETHEUS_SERVICE + - name: ELASTICSEARCH_SERVICE + valueFrom: + secretKeyRef: + name: {{ $nagiosUserSecret }} + key: ELASTICSEARCH_SERVICE + - name: NAGIOSADMIN_USER + valueFrom: + secretKeyRef: + name: {{ $nagiosUserSecret }} + key: NAGIOSADMIN_USER + - name: NAGIOSADMIN_PASS + valueFrom: + secretKeyRef: + name: {{ $nagiosUserSecret }} + key: NAGIOSADMIN_PASS +{{- if .Values.manifests.certificates }} + - name: CA_CERT_PATH + value: "/etc/ssl/certs/ca.crt" +{{- end }} + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: nagios-confd + mountPath: /opt/nagios/etc/conf.d + - name: nagios-etc + mountPath: /opt/nagios/etc/nagios.cfg + subPath: nagios.cfg + readOnly: true + - name: nagios-etc + mountPath: /opt/nagios/etc/cgi.cfg + subPath: cgi.cfg + readOnly: true + {{- $objectKeys := keys $envAll.Values.conf.nagios.objects -}} + {{- range $objectType := $objectKeys }} + - name: nagios-etc + mountPath: /opt/nagios/etc/{{$objectType}}.cfg + subPath: {{$objectType}}.cfg + readOnly: true + {{- end }} + - name: nagios-bin + mountPath: /tmp/nagios-readiness.sh + subPath: nagios-readiness.sh + readOnly: true +{{- if not (empty .Values.conf.nagios.query_es_clauses) }} + - name: nagios-etc + mountPath: /opt/nagios/etc/objects/query_es_clauses.json + subPath: query_es_clauses.json + readOnly: true +{{- end }} + - name: pod-var-log + mountPath: /opt/nagios/var/log +{{- if not (empty .Values.conf.nagios.additionalPlugins) }} +{{- range .Values.conf.nagios.additionalPlugins }} + - name: additional-plugins + mountPath: /usr/lib/nagios/plugins/{{ .name }} + subPath: {{ .name }} + readOnly: true +{{- end }} +{{- end }} +{{- dict "enabled" .Values.manifests.certificates "name" $envAll.Values.endpoints.monitoring.auth.admin.secret.tls.internal "path" "/etc/ssl/certs" "certs" tuple "ca.crt" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: pod-var-log + emptyDir: {} + - name: nagios-confd + emptyDir: {} + - name: nagios-etc + secret: + secretName: nagios-etc + defaultMode: 0444 + - name: nagios-bin + configMap: + name: nagios-bin + defaultMode: 0555 + - name: additional-plugins + secret: + secretName: nagios-additional-plugins + defaultMode: 0755 +{{- dict "enabled" .Values.manifests.certificates "name" $envAll.Values.endpoints.monitoring.auth.admin.secret.tls.internal | include "helm-toolkit.snippets.tls_volume" | indent 8 }} +{{- end }} diff --git a/nagios/templates/ingress-nagios.yaml b/nagios/templates/ingress-nagios.yaml new file mode 100644 index 0000000000..d4331ac565 --- /dev/null +++ b/nagios/templates/ingress-nagios.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.ingress .Values.network.nagios.ingress.public }} +{{- $ingressOpts := dict "envAll" . "backendService" "nagios" "backendServiceType" "nagios" "backendPort" "http" -}} +{{ $ingressOpts | include "helm-toolkit.manifests.ingress" }} +{{- end }} diff --git a/nagios/templates/job-image-repo-sync.yaml b/nagios/templates/job-image-repo-sync.yaml new file mode 100644 index 0000000000..66b0d8a751 --- /dev/null +++ b/nagios/templates/job-image-repo-sync.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "nagios" -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/nagios/templates/network_policy.yaml b/nagios/templates/network_policy.yaml new file mode 100644 index 0000000000..c31098ad78 --- /dev/null +++ b/nagios/templates/network_policy.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.network_policy -}} +{{- $netpol_opts := dict "envAll" . "name" "application" "label" "nagios" -}} +{{ $netpol_opts | include "helm-toolkit.manifests.kubernetes_network_policy" }} +{{- end -}} diff --git a/nagios/templates/pod-helm-tests.yaml b/nagios/templates/pod-helm-tests.yaml new file mode 100644 index 0000000000..0247d574fc --- /dev/null +++ b/nagios/templates/pod-helm-tests.yaml @@ -0,0 +1,81 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.pod_helm_test }} +{{- $envAll := . }} + +{{- $nagiosUserSecret := .Values.secrets.nagios.admin }} + +{{- $serviceAccountName := print .Release.Name "-test" }} +{{ tuple $envAll "tests" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: v1 +kind: Pod +metadata: + name: "{{.Release.Name}}-test" + labels: +{{ tuple $envAll "nagios" "test" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: + "helm.sh/hook": test-success + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +{{ dict "envAll" $envAll "podName" "nagios-test" "containerNames" (list "init" "nagios-helm-tests") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 4 }} +spec: +{{ dict "envAll" $envAll "application" "monitoring" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 2 }} + serviceAccountName: {{ $serviceAccountName }} + nodeSelector: + {{ .Values.labels.test.node_selector_key }}: {{ .Values.labels.test.node_selector_value }} + restartPolicy: Never + initContainers: +{{ tuple $envAll "tests" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 4 }} + containers: + - name: nagios-helm-tests +{{ tuple $envAll "selenium_tests" | include "helm-toolkit.snippets.image" | indent 6 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.tests | include "helm-toolkit.snippets.kubernetes_resources" | indent 6 }} +{{ dict "envAll" $envAll "application" "monitoring" "container" "helm_tests" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 6 }} + command: + - /tmp/selenium-tests.py + env: + - name: NAGIOS_USER + valueFrom: + secretKeyRef: + name: {{ $nagiosUserSecret }} + key: NAGIOSADMIN_USER + - name: NAGIOS_PASSWORD + valueFrom: + secretKeyRef: + name: {{ $nagiosUserSecret }} + key: NAGIOSADMIN_PASS + - name: NAGIOS_URI + value: {{ tuple "nagios" "internal" "http" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} + - name: CHROME_CONFIG_HOME + value: /tmp/google-chrome + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: artifacts + mountPath: /tmp/artifacts + - name: nagios-bin + mountPath: /tmp/selenium-tests.py + subPath: selenium-tests.py + readOnly: true + volumes: + - name: pod-tmp + emptyDir: {} + - name: artifacts + emptyDir: {} + - name: nagios-bin + configMap: + name: nagios-bin + defaultMode: 0555 +{{- end }} diff --git a/nagios/templates/secret-ingress-tls.yaml b/nagios/templates/secret-ingress-tls.yaml new file mode 100644 index 0000000000..b62b575b7e --- /dev/null +++ b/nagios/templates/secret-ingress-tls.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_ingress_tls }} +{{- include "helm-toolkit.manifests.secret_ingress_tls" ( dict "envAll" . "backendServiceType" "nagios" "backendService" "nagios" ) }} +{{- end }} diff --git a/nagios/templates/secret-nagios.yaml b/nagios/templates/secret-nagios.yaml new file mode 100644 index 0000000000..4f6fb6ade4 --- /dev/null +++ b/nagios/templates/secret-nagios.yaml @@ -0,0 +1,33 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_nagios }} +{{- $envAll := . }} +{{- $secretName := index $envAll.Values.secrets.nagios.admin }} +{{- $prometheusService := tuple "monitoring" "internal" "admin" "http" . | include "helm-toolkit.endpoints.authenticated_endpoint_uri_lookup" }} +{{- $elasticsearchService := tuple "elasticsearch" "internal" "admin" "http" . | include "helm-toolkit.endpoints.authenticated_endpoint_uri_lookup" }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} +type: Opaque +data: + NAGIOSADMIN_USER: {{ .Values.endpoints.nagios.auth.admin.username | b64enc }} + NAGIOSADMIN_PASS: {{ .Values.endpoints.nagios.auth.admin.password | b64enc }} + BIND_DN: {{ .Values.endpoints.ldap.auth.admin.bind | b64enc }} + BIND_PASSWORD: {{ .Values.endpoints.ldap.auth.admin.password | b64enc }} + PROMETHEUS_SERVICE: {{ $prometheusService | b64enc }} + ELASTICSEARCH_SERVICE: {{ $elasticsearchService | b64enc }} +{{- end }} diff --git a/nagios/templates/secret-registry.yaml b/nagios/templates/secret-registry.yaml new file mode 100644 index 0000000000..da979b3223 --- /dev/null +++ b/nagios/templates/secret-registry.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_registry .Values.endpoints.oci_image_registry.auth.enabled }} +{{ include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) }} +{{- end }} diff --git a/nagios/templates/service-ingress-nagios.yaml b/nagios/templates/service-ingress-nagios.yaml new file mode 100644 index 0000000000..9af4ec329e --- /dev/null +++ b/nagios/templates/service-ingress-nagios.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.service_ingress .Values.network.nagios.ingress.public }} +{{- $serviceIngressOpts := dict "envAll" . "backendServiceType" "nagios" -}} +{{ $serviceIngressOpts | include "helm-toolkit.manifests.service_ingress" }} +{{- end }} diff --git a/nagios/templates/service.yaml b/nagios/templates/service.yaml new file mode 100644 index 0000000000..4789283121 --- /dev/null +++ b/nagios/templates/service.yaml @@ -0,0 +1,34 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.service }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "nagios" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +spec: + ports: + - name: http + port: {{ tuple "nagios" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + {{ if .Values.network.nagios.node_port.enabled }} + nodePort: {{ .Values.network.nagios.node_port.port }} + {{ end }} + selector: +{{ tuple $envAll "nagios" "monitoring" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + {{ if .Values.network.nagios.node_port.enabled }} + type: NodePort + {{ end }} +{{- end }} diff --git a/nagios/values.yaml b/nagios/values.yaml new file mode 100644 index 0000000000..2726e5a81d --- /dev/null +++ b/nagios/values.yaml @@ -0,0 +1,1230 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for nagios. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +--- +images: + tags: + apache_proxy: docker.io/library/httpd:2.4 + nagios: docker.io/openstackhelm/nagios:latest-ubuntu_jammy + dep_check: quay.io/stackanetes/kubernetes-entrypoint:v0.2.1 + selenium_tests: docker.io/openstackhelm/osh-selenium:latest-ubuntu_jammy + image_repo_sync: docker.io/library/docker:17.07.0 + pull_policy: IfNotPresent + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +# Use selenium v4 syntax +selenium_v4: true + +labels: + nagios: + node_selector_key: openstack-control-plane + node_selector_value: enabled + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + test: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +dependencies: + dynamic: + common: + jobs: + - nagios-image-repo-sync + services: + - service: local_image_registry + endpoint: node + static: + image_repo_sync: + services: + - service: local_image_registry + endpoint: internal + nagios: + services: null + tests: + services: + - service: nagios + endpoint: internal + +secrets: + nagios: + admin: nagios-admin-creds + oci_image_registry: + nagios: nagios-oci-image-registry-key + tls: + nagios: + nagios: + public: nagios-tls-public + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + nagios: + username: nagios + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + monitoring: + name: prometheus + auth: + admin: + username: admin + password: changeme + secret: + tls: + internal: prometheus-tls-api + hosts: + default: prom-metrics + public: prometheus + host_fqdn_override: + default: null + path: + default: null + scheme: + default: http + port: + http: + default: 80 + nagios: + name: nagios + namespace: null + auth: + admin: + username: nagiosadmin + password: password + hosts: + default: nagios-metrics + public: nagios + host_fqdn_override: + default: null + # NOTE(srwilkers): this chart supports TLS for fqdn over-ridden public + # endpoints using the following format: + # public: + # host: null + # tls: + # crt: null + # key: null + path: + default: null + scheme: + default: http + port: + nagios: + default: 8000 + http: + default: 80 + ldap: + hosts: + default: ldap + auth: + admin: + bind: "cn=admin,dc=cluster,dc=local" + password: password + host_fqdn_override: + default: null + path: + default: "/ou=People,dc=cluster,dc=local" + scheme: + default: ldap + port: + ldap: + default: 389 + elasticsearch: + name: elasticsearch + namespace: null + auth: + admin: + username: admin + password: changeme + hosts: + default: elasticsearch-logging + host_fqdn_override: + default: null + path: + default: / + scheme: + default: http + port: + http: + default: 80 + ceph_mgr: + namespace: null + hosts: + default: ceph-mgr + host_fqdn_override: + default: null + port: + mgr: + default: 7000 + metrics: + default: 9283 + scheme: + default: http + +network: + nagios: + ingress: + public: true + classes: + namespace: "nginx" + cluster: "nginx-cluster" + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / + nginx.ingress.kubernetes.io/affinity: cookie + nginx.ingress.kubernetes.io/session-cookie-name: kube-ingress-session-nagios + nginx.ingress.kubernetes.io/session-cookie-hash: sha1 + nginx.ingress.kubernetes.io/session-cookie-expires: "600" + nginx.ingress.kubernetes.io/session-cookie-max-age: "600" + nginx.ingress.kubernetes.io/configuration-snippet: | + more_set_headers "X-Content-Type-Options: 'nosniff'"; + more_set_headers "X-Frame-Options: SAMEORIGIN"; + more_set_headers "Content-Security-Policy: script-src 'self'"; + more_set_headers "X-XSS-Protection: 1; mode=block"; + node_port: + enabled: false + port: 30925 + +network_policy: + nagios: + ingress: + - {} + egress: + - {} + +pod: + security_context: + monitoring: + pod: + runAsUser: 0 + container: + define_nagios_hosts: + readOnlyRootFilesystem: false + apache_proxy: + readOnlyRootFilesystem: false + nagios: + readOnlyRootFilesystem: false + helm_tests: + readOnlyRootFilesystem: true + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + weight: + default: 10 + lifecycle: + upgrades: + deployments: + revision_history: 3 + pod_replacement_strategy: RollingUpdate + rolling_update: + max_unavailable: 1 + max_surge: 3 + termination_grace_period: + nagios: + timeout: 30 + # env: + # + # NOTE(megheisler): This value can be used to hold + # the domain name. Functionality has been added in + # plugins to append the domain to the host name in + # the nagios dashboard + # + # NODE_DOMAIN: + replicas: + nagios: 1 + probes: + monitoring: + nagios: + readiness: + enabled: true + params: + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 10 + apache_proxy: + readiness: + enabled: true + params: + initialDelaySeconds: 20 + periodSeconds: 10 + resources: + enabled: false + nagios: + limits: + memory: "1024Mi" + cpu: "2000m" + requests: + memory: "128Mi" + cpu: "100m" + apache_proxy: + limits: + memory: "1024Mi" + cpu: "2000m" + requests: + memory: "128Mi" + cpu: "100m" + jobs: + image_repo_sync: + limits: + memory: "1024Mi" + cpu: "2000m" + requests: + memory: "128Mi" + cpu: "100m" + tests: + limits: + memory: "1024Mi" + cpu: "2000m" + requests: + memory: "128Mi" + cpu: "100m" + +manifests: + certificates: false + configmap_additional_plugins: false + configmap_bin: true + configmap_etc: true + deployment: true + ingress: true + job_image_repo_sync: true + network_policy: false + pod_helm_test: true + secret_nagios: true + secret_ingress_tls: true + secret_registry: true + service: true + service_ingress: true + +conf: + httpd: | + ServerRoot "/usr/local/apache2" + + Listen 80 + + LoadModule mpm_event_module modules/mod_mpm_event.so + LoadModule authn_file_module modules/mod_authn_file.so + LoadModule authn_core_module modules/mod_authn_core.so + LoadModule authz_host_module modules/mod_authz_host.so + LoadModule authz_groupfile_module modules/mod_authz_groupfile.so + LoadModule authz_user_module modules/mod_authz_user.so + LoadModule authz_core_module modules/mod_authz_core.so + LoadModule access_compat_module modules/mod_access_compat.so + LoadModule auth_basic_module modules/mod_auth_basic.so + LoadModule ldap_module modules/mod_ldap.so + LoadModule authnz_ldap_module modules/mod_authnz_ldap.so + LoadModule reqtimeout_module modules/mod_reqtimeout.so + LoadModule filter_module modules/mod_filter.so + LoadModule proxy_html_module modules/mod_proxy_html.so + LoadModule log_config_module modules/mod_log_config.so + LoadModule env_module modules/mod_env.so + LoadModule headers_module modules/mod_headers.so + LoadModule setenvif_module modules/mod_setenvif.so + LoadModule version_module modules/mod_version.so + LoadModule proxy_module modules/mod_proxy.so + LoadModule proxy_connect_module modules/mod_proxy_connect.so + LoadModule proxy_http_module modules/mod_proxy_http.so + LoadModule proxy_balancer_module modules/mod_proxy_balancer.so + LoadModule slotmem_shm_module modules/mod_slotmem_shm.so + LoadModule slotmem_plain_module modules/mod_slotmem_plain.so + LoadModule unixd_module modules/mod_unixd.so + LoadModule status_module modules/mod_status.so + LoadModule autoindex_module modules/mod_autoindex.so + + + User daemon + Group daemon + + + + AllowOverride none + Require all denied + + + + Require all denied + + + ErrorLog /dev/stderr + + LogLevel warn + + + LogFormat "%a %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined + LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" proxy + LogFormat "%h %l %u %t \"%r\" %>s %b" common + + + LogFormat "%a %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combinedio + + + SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded + CustomLog /dev/stdout common + CustomLog /dev/stdout combined + CustomLog /dev/stdout proxy env=forwarded + + + + AllowOverride None + Options None + Require all granted + + + + RequestHeader unset Proxy early + + + + Include conf/extra/proxy-html.conf + + + + + ProxyPass http://localhost:{{ tuple "nagios" "internal" "nagios" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/ + ProxyPassReverse http://localhost:{{ tuple "nagios" "internal" "nagios" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/ + + + AuthName "Nagios" + AuthType Basic + AuthBasicProvider file ldap + AuthUserFile /usr/local/apache2/conf/.htpasswd + AuthLDAPBindDN {{ .Values.endpoints.ldap.auth.admin.bind }} + AuthLDAPBindPassword {{ .Values.endpoints.ldap.auth.admin.password }} + AuthLDAPURL {{ tuple "ldap" "default" "ldap" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | quote }} + Require valid-user + + + nagios: + notification: + snmp: + primary_target: 127.0.0.1:15162 + secondary_target: 127.0.0.1:15162 + http: + primary_target: 127.0.0.1:3904/events + secondary_target: 127.0.0.1:3904/events + objects: + base: + template: | + define host { + address 127.0.0.1 + alias Prometheus Monitoring + check_command check-prometheus-host-alive + host_name {{ tuple "monitoring" "public" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + hostgroups prometheus-hosts + use linux-server + } + + define contact { + alias notifying contact + contact_name notifying_contact + host_notification_options d,u,r,f,s + host_notification_period 24x7 + name notifying_contact + register 0 + service_notification_options w,u,c,r,f,s + service_notification_period 24x7 + } + + define contact { + alias snmp contact + contact_name snmp_notifying_contact + host_notification_commands send_host_snmp_trap + name snmp_notifying_contact + service_notification_commands send_service_snmp_trap + use notifying_contact + } + + define contact { + alias HTTP contact + contact_name http_notifying_contact + host_notification_commands send_host_http_post + name http_notifying_contact + service_notification_commands send_service_http_post + use notifying_contact + } + + define contactgroup { + alias SNMP and HTTP notifying group + contactgroup_name snmp_and_http_notifying_contact_group + members snmp_notifying_contact,http_notifying_contact + } + + define hostgroup { + alias Prometheus Virtual Host + hostgroup_name prometheus-hosts + } + + define hostgroup { + alias all + hostgroup_name all + } + + define hostgroup { + alias base-os + hostgroup_name base-os + } + + define command { + command_line $USER1$/send_service_trap.sh '$USER8$' '$HOSTNAME$' '$SERVICEDESC$' $SERVICESTATEID$ '$SERVICEOUTPUT$' '$USER4$' '$USER5$' + command_name send_service_snmp_trap + } + + define command { + command_line $USER1$/send_host_trap.sh '$USER8$' '$HOSTNAME$' $HOSTSTATEID$ '$HOSTOUTPUT$' '$USER4$' '$USER5$' + command_name send_host_snmp_trap + } + + define command { + command_line $USER1$/send_http_post_event.py --type service --hostname '$HOSTNAME$' --servicedesc '$SERVICEDESC$' --state_id $SERVICESTATEID$ --output '$SERVICEOUTPUT$' --monitoring_hostname '$HOSTNAME$' --primary_url '$USER6$' --secondary_url '$USER7$' + command_name send_service_http_post + } + + define command { + command_line $USER1$/send_http_post_event.py --type host --hostname '$HOSTNAME$' --state_id $HOSTSTATEID$ --output '$HOSTOUTPUT$' --monitoring_hostname '$HOSTNAME$' --primary_url '$USER6$' --secondary_url '$USER7$' + command_name send_host_http_post + } + + define command { + command_line $USER1$/check_rest_get_api.py --url $USER2$ --warning_response_seconds 5 --critical_response_seconds 10 + command_name check-prometheus-host-alive + } + + define command { + command_line $USER1$/query_prometheus_alerts.py --prometheus_api $USER2$ --alertname '$ARG1$' --labels_csv '$ARG2$' --msg_format '$ARG3$' --ok_message '$ARG4$' + command_name check_prom_alert_with_labels + } + + define command { + command_line $USER1$/query_prometheus_alerts.py --prometheus_api $USER2$ --alertname '$ARG1$' --msg_format '$ARG2$' --ok_message '$ARG3$' + command_name check_prom_alert + } + + define service { + check_interval 60 + contact_groups snmp_and_http_notifying_contact_group + flap_detection_enabled 0 + name notifying_service + notification_interval 120 + process_perf_data 0 + register 0 + retry_interval 30 + use generic-service + } + kubernetes: + template: | + define service { + check_command check_prom_alert!prom_exporter_calico_unavailable!CRITICAL- Calico exporter is not collecting metrics for alerting!OK- Calico exporter metrics are available. + hostgroup_name prometheus-hosts + service_description Prometheus-exporter_Calico + use generic-service + } + + define service { + check_command check_prom_alert!prom_exporter_kube_state_metrics_unavailable!CRITICAL- kube-state-metrics exporter is not collecting metrics for alerting!OK- kube-state-metrics exporter metrics are available. + hostgroup_name prometheus-hosts + service_description Prometheus-exporter_Kube-state-metrics + use generic-service + } + + define service { + check_command check_prom_alert!K8SNodesNotReady!CRITICAL- One or more nodes are not ready. + check_interval 60 + hostgroup_name prometheus-hosts + service_description Nodes_health + use generic-service + } + + define service { + check_command check_prom_alert_with_labels!kube_statefulset_replicas_unavailable!statefulset="prometheus"!statefulset {statefulset} has lesser than configured replicas + check_interval 60 + hostgroup_name prometheus-hosts + service_description Prometheus_replica-count + use notifying_service + } + + define service { + check_command check_prom_alert_with_labels!kube_statefulset_replicas_unavailable!statefulset="alertmanager"!statefulset {statefulset} has lesser than configured replicas + check_interval 60 + hostgroup_name prometheus-hosts + service_description PrometheusAlertmanager_replica-count + use notifying_service + } + + define service { + check_command check_prom_alert!kube_statefulset_replicas_unavailable!CRITICAL- statefulset {statefulset} has lesser than configured replicas!OK- All statefulsets have configured amount of replicas + check_interval 60 + hostgroup_name prometheus-hosts + service_description Statefulset_replica-count + use notifying_service + } + + define service { + check_command check_prom_alert!daemonsets_misscheduled!CRITICAL- Daemonset {daemonset} is incorrectly scheudled!OK- No daemonset misscheduling detected + check_interval 60 + hostgroup_name prometheus-hosts + service_description Daemonset_misscheduled + use notifying_service + } + + define service { + check_command check_prom_alert!daemonsets_not_scheduled!CRITICAL- Daemonset {daemonset} is missing to be scheduled in some nodes!OK- All daemonset scheduling is as desired + check_interval 60 + hostgroup_name prometheus-hosts + service_description Daemonset_not-scheduled + use notifying_service + } + + define service { + check_command check_prom_alert!daemonset_pods_unavailable!CRITICAL- Daemonset {daemonset} has pods unavailable!OK- All daemonset pods available + check_interval 60 + hostgroup_name prometheus-hosts + service_description Daemonset_pods-unavailable + use notifying_service + } + + define service { + check_command check_prom_alert!deployment_replicas_unavailable!CRITICAL- Deployment {deployment} has less than desired replicas!OK- All deployments have desired replicas + check_interval 60 + hostgroup_name prometheus-hosts + service_description Deployment_replicas-unavailable + use notifying_service + } + + define service { + check_command check_prom_alert!volume_claim_capacity_high_utilization!CRITICAL- Volume claim {persistentvolumeclaim} has exceed 80% utilization!OK- All volume claims less than 80% utilization + check_interval 60 + hostgroup_name prometheus-hosts + service_description Volume_claim_high_utilization + use notifying_service + } + + define service { + check_command check_prom_alert!rollingupdate_deployment_replica_less_than_spec_max_unavailable!CRITICAL- Deployment {deployment} has less than desired replicas during a rolling update!OK- All deployments have desired replicas + check_interval 60 + hostgroup_name prometheus-hosts + service_description RollingUpdate_Deployment-replicas-unavailable + use notifying_service + } + + define service { + check_command check_prom_alert!job_status_failed!CRITICAL- Job {exported_job} has failed!OK- No Job failures + check_interval 60 + hostgroup_name prometheus-hosts + service_description Job_status-failed + use notifying_service + } + + define service { + check_command check_prom_alert!pod_status_pending!CRITICAL- Pod {pod} in namespace {namespace} has been in pending status for more than 10 minutes!OK- No pods in pending status + check_interval 60 + hostgroup_name prometheus-hosts + service_description Pod_status-pending + use notifying_service + } + + define service { + check_command check_prom_alert!pod_status_error_image_pull!CRITICAL- Pod {pod} in namespace {namespace} has been in errpr status of ErrImagePull for more than 10 minutes!OK- No pods in error status + check_interval 60 + hostgroup_name prometheus-hosts + service_description Pod_status-error-image-pull + use notifying_service + } + + define service { + check_command check_prom_alert! pod_status_error_image_pull_backoff!CRITICAL- Pod {pod} in namespace {namespace} has been in errpr status of ImagePullBackOff for more than 10 minutes!OK- No pods in error status + check_interval 60 + hostgroup_name prometheus-hosts + service_description Pod_status-error-image-pull + use notifying_service + } + + define service { + check_command check_prom_alert! pod_error_config_error!CRITICAL- Pod {pod} in namespace {namespace} has been in errpr status of CreateContainerConfigError for more than 10 minutes!OK- No pods in error status + check_interval 60 + hostgroup_name prometheus-hosts + service_description Pod_status-error-image-pull + use notifying_service + } + + define service { + check_command check_prom_alert!pod_error_crash_loop_back_off!CRITICAL- Pod {pod} in namespace {namespace} has been in error status of CrashLoopBackOff for more than 10 minutes!OK- No pods in crashLoopBackOff status + check_interval 60 + hostgroup_name prometheus-hosts + service_description Pod_status-crashLoopBackOff + use notifying_service + } + + define service { + check_command check_prom_alert!replicaset_missing_replicas!CRITICAL- Replicaset {replicaset} is missing replicas!OK- No replicas missing from replicaset + check_interval 60 + hostgroup_name prometheus-hosts + service_description Replicaset_missing-replicas + use notifying_service + } + + define service { + check_command check_prom_alert!pod_container_terminated!CRITICAL- pod {pod} in namespace {namespace} has a container in terminated state!OK- pod container status looks good + check_interval 60 + hostgroup_name prometheus-hosts + service_description Pod_status-container-terminated + use notifying_service + } + + define service { + check_command check_prom_alert_with_labels!etcd_HighNumberOfFailedHTTPRequests!method="DELETE"!CRITICAL- ETCD {instance} has a high HTTP DELETE operations failure!OK- ETCD at {instance} has low or no failures for HTTP DELETE + check_interval 60 + hostgroup_name prometheus-hosts + service_description ETCD_high-http-delete-failures + use notifying_service + } + + define service { + check_command check_prom_alert_with_labels!etcd_HighNumberOfFailedHTTPRequests!method=~"GET|QGET"!CRITICAL- ETCD {instance} has a high HTTP GET operations failure!OK- ETCD at {instance} has low or no failures for HTTP GET + check_interval 60 + hostgroup_name prometheus-hosts + service_description ETCD_high-http-get-failures + use notifying_service + } + + define service { + check_command check_prom_alert_with_labels!etcd_HighNumberOfFailedHTTPRequests!method="PUT"!CRITICAL- ETCD {instance} has a high HTTP PUT operations failure!OK- ETCD at {instance} has low or no failures for HTTP PUT + check_interval 60 + hostgroup_name prometheus-hosts + service_description ETCD_high-http-update-failures + use notifying_service + } + + define service { + check_command check_prom_alert!calico_iptable_save_errors_high_1h!CRITICAL- Felix instance {instance} has seen high iptable save errors within the last hour!OK- iptables save errors are none or low + hostgroup_name prometheus-hosts + service_description Calico_iptables-save-errors + use notifying_service + } + + define service { + check_command check_prom_alert!calico_ipset_errors_high_1h!CRITICAL- Felix instance {instance} has seen high ipset errors within the last hour!OK- ipset errors are none or low + hostgroup_name prometheus-hosts + service_description Calico_ipset-errors + use notifying_service + } + + define service { + check_command check_prom_alert!calico_datapane_iface_msg_batch_size_high_5m!CRITICAL- Felix instance {instance} has seen a high value of dataplane interface message batch size!OK- dataplane interface message batch size are low + hostgroup_name prometheus-hosts + service_description Calico_interface-message-batch-size + use notifying_service + } + + define service { + check_command check_prom_alert!calico_datapane_address_msg_batch_size_high_5m!CRITICAL- Felix instance {instance} has seen a high value of dataplane address message batch size!OK- dataplane address message batch size are low + hostgroup_name prometheus-hosts + service_description Calico_address-message-batch-size + use notifying_service + } + + define service { + check_command check_prom_alert!calico_datapane_failures_high_1h!CRITICAL- Felix instance {instance} has seen high dataplane failures within the last hour!OK- datapane failures are none or low + hostgroup_name prometheus-hosts + service_description Calico_datapane_failures_high + use notifying_service + } + node: + template: | + define service { + check_command check_prom_alert!prom_exporter_node_unavailable!CRITICAL- Node exporter is not collecting metrics for alerting!OK- Node exporter metrics are available. + hostgroup_name prometheus-hosts + service_description Prometheus-exporter_Node + use generic-service + } + + define command { + command_line $USER1$/query_prometheus_alerts.py --prometheus_api $USER2$ --alertname 'node_filesystem_full_in_4h' --labels_csv 'instance=~"$HOSTADDRESS$.*"' --msg_format 'CRITICAL- Mountpoint {mountpoint} will be full in four hours' --ok_message 'OK- All mountpoints usage rate is normal' + command_name check_filespace_mounts-usage-rate-fullin4hrs + } + + define command { + command_line $USER1$/query_prometheus_alerts.py --prometheus_api $USER2$ --alertname 'node_filesystem_full_80percent' --labels_csv 'instance=~"$HOSTADDRESS$.*"' --msg_format 'CRITICAL- Mountpoint {mountpoint} is more than 80 pecent full' --ok_message 'OK- All mountpoints usage is normal' + command_name check_filespace_mounts-usage + } + + define command { + command_line $USER1$/query_prometheus_alerts.py --prometheus_api $USER2$ --alertname 'node_load1_90percent' --labels_csv 'instance=~"$HOSTADDRESS$.*"' --msg_format 'CRITICAL- Node load average has been more than 90% for the pash hour' --ok_message 'OK- Node load average is normal' + command_name check_node_loadavg + } + + define command { + command_line $USER1$/query_prometheus_alerts.py --prometheus_api $USER2$ --alertname 'node_cpu_util_90percent' --labels_csv 'instance=~"$HOSTADDRESS$.*"' --msg_format 'CRITICAL- Node CPU utilization has been more than 90% for the pash hour' --ok_message 'OK- Node cpu utilization is normal' + command_name check_node_cpu_util + } + + define command { + command_line $USER1$/query_prometheus_alerts.py --prometheus_api $USER2$ --alertname 'node_network_conntrack_usage_80percent' --labels_csv 'instance=~"$HOSTADDRESS$.*"' --msg_format 'CRITICAL- Node network connections are more than 90% in use' --ok_message 'OK- Network connection utilization is normal' + command_name check_network_connections + } + + define command { + command_line $USER1$/query_prometheus_alerts.py --prometheus_api $USER2$ --alertname 'node_high_memory_load' --labels_csv 'instance=~"$HOSTADDRESS$.*"' --msg_format 'CRITICAL- Node memory usage is more than 85%' --ok_message 'OK- Node memory usage is less than 85%' + command_name check_memory_usage + } + + define command { + command_line $USER1$/query_prometheus_alerts.py --prometheus_api $USER2$ --alertname 'node_disk_write_latency' --labels_csv 'instance=~"$HOSTADDRESS$.*"' --msg_format 'CRITICAL- Disk write latency is high' --ok_message 'OK- Node disk write latency is normal' + command_name check_disk_write_latency + } + + define command { + command_line $USER1$/query_prometheus_alerts.py --prometheus_api $USER2$ --alertname 'node_disk_read_latency' --labels_csv 'instance=~"$HOSTADDRESS$.*"' --msg_format 'CRITICAL- Disk read latency is high' --ok_message 'OK- Node disk read latency is normal' + command_name check_disk_read_latency + } + + define command { + command_line $USER1$/query_prometheus_alerts.py --prometheus_api $USER2$ --alertname 'node_entropy_available_low' --labels_csv 'instance=~"$HOSTADDRESS$.*"' --msg_format 'CRITICAL- System has low entropy availability' --ok_message 'OK- System entropy availability is sufficient' + command_name check_entropy_availability + } + + define command { + command_line $USER1$/query_prometheus_alerts.py --prometheus_api $USER2$ --alertname 'node_filedescriptors_full_in_3h' --labels_csv 'instance=~"$HOSTADDRESS$.*"' --msg_format 'CRITICAL- at current consumption rate no free file descriptors will be available in 3hrs.' --ok_message 'OK- System file descriptor consumption is ok.' + command_name check_filedescriptor_usage_rate + } + + define command { + command_line $USER1$/query_prometheus_alerts.py --prometheus_api $USER2$ --alertname 'node_hwmon_high_cpu_temp' --labels_csv 'instance=~"$HOSTADDRESS$.*"' --msg_format 'CRITICAL- CPU temperature is 90 percent of critical temperature.' --ok_message 'OK- CPU temperatures are normal.' + command_name check_hwmon_high_cpu_temp + } + + define command { + command_line $USER1$/query_prometheus_alerts.py --prometheus_api $USER2$ --alertname 'node_high_network_drop_rcv' --labels_csv 'instance=~"$HOSTADDRESS$.*"' --msg_format 'CRITICAL- Host system has an unusally high drop in network reception.' --ok_message 'OK- network packet receive drops not high.' + command_name check_network_receive_drop_high + } + + define command { + command_line $USER1$/query_prometheus_alerts.py --prometheus_api $USER2$ --alertname 'node_high_network_drop_send' --labels_csv 'instance=~"$HOSTADDRESS$.*"' --msg_format 'CRITICAL- Host system has an unusally high drop in network transmission.' --ok_message 'OK- network packet tramsmit drops not high.' + command_name check_network_transmit_drop_high + } + + define command { + command_line $USER1$/query_prometheus_alerts.py --prometheus_api $USER2$ --alertname 'node_high_network_errs_rcv' --labels_csv 'instance=~"$HOSTADDRESS$.*"' --msg_format 'CRITICAL- Host system has an unusally high error rate in network reception.' --ok_message 'OK- network reception errors not high.' + command_name check_network_receive_errors_high + } + + define command { + command_line $USER1$/query_prometheus_alerts.py --prometheus_api $USER2$ --alertname 'node_high_network_errs_send' --labels_csv 'instance=~"$HOSTADDRESS$.*"' --msg_format 'CRITICAL- Host system has an unusally high error rate in network transmission.' --ok_message 'OK- network transmission errors not high.' + command_name check_network_transmit_errors_high + } + + define command { + command_line $USER1$/query_prometheus_alerts.py --prometheus_api $USER2$ --alertname 'node_vmstat_paging_rate_high' --labels_csv 'instance=~"$HOSTADDRESS$.*"' --msg_format 'CRITICAL- Memory paging rate over 5 minutes is high.' --ok_message 'OK- Memory paging rate over 5 minutes is ok.' + command_name check_vmstat_paging_rate + } + + define command { + command_line $USER1$/query_prometheus_alerts.py --prometheus_api $USER2$ --alertname 'node_xfs_block_allocation_high' --labels_csv 'instance=~"$HOSTADDRESS$.*"' --msg_format 'CRITICAL- XFS block allocation is more than 80 percent of available.' --ok_message 'OK- XFS block allocation is less than 80 percent of available.' + command_name check_xfs_block_allocation + } + + define command { + command_line $USER1$/query_prometheus_alerts.py --prometheus_api $USER2$ --alertname 'node_network_bond_slaves_down' --labels_csv 'instance=~"$HOSTADDRESS$.*"' --msg_format 'CRITICAL- {master} is missing slave interfaces.' --ok_message 'OK- Network bonds have slave interfaces functional.' + command_name check_network_bond_status + } + + define command { + command_line $USER1$/query_prometheus_alerts.py --prometheus_api $USER2$ --alertname 'node_numa_memory_used' --labels_csv 'instance=~"$HOSTADDRESS$.*"' --msg_format 'CRITICAL- NUMA memory usage is more than 80 percent of available.' --ok_message 'OK- NUMA memory usage is normal.' + command_name check_numa_memory_usage + } + + define command { + command_line $USER1$/query_prometheus_alerts.py --prometheus_api $USER2$ --alertname 'node_ntp_clock_skew_high' --labels_csv 'instance=~"$HOSTADDRESS$.*"' --msg_format 'CRITICAL- NTP clock skew is more than 2 seconds.' --ok_message 'OK- NTP clock skew is less than 2 seconds.' + command_name check_ntp_sync + } + + define service { + check_command check_filespace_mounts-usage-rate-fullin4hrs + check_interval 60 + hostgroup_name base-os + service_description Filespace_mounts-usage-rate-fullin4hrs + use notifying_service + } + + define service { + check_command check_filespace_mounts-usage + check_interval 60 + hostgroup_name base-os + service_description Filespace_mounts-usage + use notifying_service + } + + define service { + check_command check_node_loadavg + hostgroup_name base-os + service_description CPU_Load-average + use notifying_service + } + + define service { + check_command check_node_cpu_util + hostgroup_name base-os + service_description CPU_utilization + use notifying_service + } + + define service { + check_command check_network_connections + hostgroup_name base-os + service_description Network_connections + use notifying_service + } + + define service { + check_command check_memory_usage + hostgroup_name base-os + service_description Memory_usage + use notifying_service + } + + define service { + check_command check_disk_write_latency + hostgroup_name base-os + service_description Disk_write-latency + use notifying_service + } + + define service { + check_command check_disk_read_latency + hostgroup_name base-os + service_description Disk_read-latency + use notifying_service + } + + define service { + check_command check_entropy_availability + hostgroup_name base-os + service_description Entropy_availability + use notifying_service + } + + define service { + check_command check_filedescriptor_usage_rate + hostgroup_name base-os + service_description FileDescriptors_usage-rate-high + use notifying_service + } + + define service { + check_command check_hwmon_high_cpu_temp + hostgroup_name base-os + service_description HW_cpu-temp-high + use notifying_service + } + + define service { + check_command check_network_receive_drop_high + hostgroup_name base-os + service_description Network_receive-drop-high + use notifying_service + } + + define service { + check_command check_network_transmit_drop_high + hostgroup_name base-os + service_description Network_transmit-drop-high + use notifying_service + } + + define service { + check_command check_network_receive_errors_high + hostgroup_name base-os + service_description Network_receive-errors-high + use notifying_service + } + + define service { + check_command check_network_transmit_errors_high + hostgroup_name base-os + service_description Network_transmit-errors-high + use notifying_service + } + + define service { + check_command check_vmstat_paging_rate + hostgroup_name base-os + service_description Memory_vmstat-paging-rate + use notifying_service + } + + define service { + check_command check_xfs_block_allocation + hostgroup_name base-os + service_description XFS_block-allocation + use notifying_service + } + + define service { + check_command check_network_bond_status + hostgroup_name base-os + service_description Network_bondstatus + use notifying_service + } + + define service { + check_command check_numa_memory_usage + hostgroup_name base-os + service_description Memory_NUMA-usage + use notifying_service + } + + define service { + check_command check_ntp_sync + hostgroup_name base-os + service_description NTP_sync + use notifying_service + } + ceph: + template: | + define service { + check_command check_prom_alert!prom_exporter_ceph_unavailable!CRITICAL- CEPH exporter is not collecting metrics for alerting!OK- CEPH exporter metrics are available. + hostgroup_name prometheus-hosts + service_description Prometheus-exporter_CEPH + use generic-service + } + + define command { + command_line $USER1$/check_exporter_health_metric.py --exporter_api $USER10$ --health_metric ceph_health_status --critical 2 --warning 1 + command_name check_ceph_health + } + + define service { + check_command check_ceph_health + check_interval 300 + hostgroup_name base-os + service_description CEPH_health + use notifying_service + } + + define service { + check_command check_prom_alert!ceph_monitor_quorum_low!CRITICAL- ceph monitor quorum does not exist!OK- ceph monitor quorum exists + check_interval 60 + hostgroup_name prometheus-hosts + service_description CEPH_quorum + use notifying_service + } + + define service { + check_command check_prom_alert!ceph_monitor_quorum_absent!CRITICAL- ceph monitor quorum does not exist!OK- ceph monitor quorum exists + check_interval 60 + hostgroup_name prometheus-hosts + service_description CEPH_quorum + use notifying_service + } + + define service { + check_command check_prom_alert!ceph_cluster_usage_high!CRITICAL- ceph cluster storage is more than 80 percent!OK- ceph storage is less than 80 percent + check_interval 60 + hostgroup_name prometheus-hosts + service_description CEPH_storage-usage + use notifying_service + } + + define service { + check_command check_prom_alert!ceph_placement_group_degrade_pct_high!CRITICAL- ceph cluster PGs down are more than 80 percent!OK- ceph PG degradation is less than 80 percent + check_interval 60 + hostgroup_name prometheus-hosts + service_description CEPH_PGs-degradation + use notifying_service + } + + define service { + check_command check_prom_alert!ceph_osd_down!CRITICAL- One or more CEPH OSDs are down for more than 5 minutes!OK- All the CEPH OSDs are up + check_interval 60 + hostgroup_name prometheus-hosts + service_description CEPH_OSDs-down + use notifying_service + } + + define service { + check_command check_prom_alert_with_labels!node_ntp_clock_skew_high!ceph-mon="enabled"!CRITICAL- CEPH clock skew is more than 2 seconds!OK- CEPH clock skew is less than 2 seconds + check_interval 60 + hostgroup_name prometheus-hosts + service_description CEPH_Clock-skew + use notifying_service + } + nagios: + template: | + accept_passive_host_checks=1 + accept_passive_service_checks=1 + additional_freshness_latency=15 + allow_empty_hostgroup_assignment=1 + auto_reschedule_checks=0 + auto_rescheduling_interval=30 + auto_rescheduling_window=180 + bare_update_check=0 + cached_host_check_horizon=15 + cached_service_check_horizon=15 + {{- $objectKeys := keys .Values.conf.nagios.objects -}} + {{- range $object := $objectKeys }} + cfg_file=/opt/nagios/etc/{{$object}}.cfg + {{- end }} + cfg_file=/opt/nagios/etc/objects/commands.cfg + cfg_file=/opt/nagios/etc/objects/contacts.cfg + cfg_file=/opt/nagios/etc/objects/timeperiods.cfg + cfg_file=/opt/nagios/etc/objects/templates.cfg + cfg_file=/opt/nagios/etc/conf.d/nagios-hosts.cfg + + check_external_commands=1 + check_for_orphaned_hosts=1 + check_for_orphaned_services=1 + check_for_updates=1 + check_host_freshness=0 + check_result_path=/opt/nagios/var/spool/checkresults + check_result_reaper_frequency=10 + check_service_freshness=1 + check_workers=4 + command_file=/opt/nagios/var/rw/nagios.cmd + daemon_dumps_core=0 + date_format=us + debug_file=/opt/nagios/var/nagios.debug + debug_level=0 + debug_verbosity=1 + enable_environment_macros=0 + enable_event_handlers=1 + enable_flap_detection=1 + enable_notifications=1 + enable_predictive_host_dependency_checks=1 + enable_predictive_service_dependency_checks=1 + event_broker_options=-1 + event_handler_timeout=60 + execute_host_checks=1 + execute_service_checks=1 + high_host_flap_threshold=20 + high_service_flap_threshold=20 + host_check_timeout=60 + host_freshness_check_interval=60 + host_inter_check_delay_method=s + illegal_macro_output_chars=`~$&|'<>" + interval_length=1 + lock_file=/var/run/nagios.lock + log_archive_path=/opt/nagios/var/log/archives + log_current_states=1 + log_event_handlers=1 + log_external_commands=1 + log_file=/opt/nagios/var/log/nagios.log + log_host_retries=1 + log_initial_states=0 + log_notifications=0 + log_passive_checks=1 + log_rotation_method=d + log_service_retries=1 + low_host_flap_threshold=5 + low_service_flap_threshold=5 + max_check_result_file_age=3600 + max_check_result_reaper_time=30 + max_concurrent_checks=10 + max_debug_file_size=1e+06 + max_host_check_spread=30 + max_service_check_spread=30 + nagios_group=nagios + nagios_user=nagios + notification_timeout=60 + object_cache_file=/opt/nagios/var/objects.cache + obsess_over_hosts=0 + obsess_over_services=0 + ocsp_timeout=5 + passive_host_checks_are_soft=0 + perfdata_timeout=5 + precached_object_file=/opt/nagios/var/objects.precache + process_performance_data=0 + resource_file=/opt/nagios/etc/resource.cfg + retain_state_information=1 + retained_contact_host_attribute_mask=0 + retained_contact_service_attribute_mask=0 + retained_host_attribute_mask=0 + retained_process_host_attribute_mask=0 + retained_process_service_attribute_mask=0 + retained_service_attribute_mask=0 + retention_update_interval=60 + service_check_timeout=60 + service_freshness_check_interval=60 + service_inter_check_delay_method=s + service_interleave_factor=s + soft_state_dependencies=0 + state_retention_file=/opt/nagios/var/retention.dat + status_file=/opt/nagios/var/status.dat + status_update_interval=10 + temp_file=/opt/nagios/var/nagios.tmp + temp_path=/tmp + translate_passive_host_checks=0 + use_aggressive_host_checking=0 + use_large_installation_tweaks=0 + use_regexp_matching=1 + use_retained_program_state=1 + use_retained_scheduling_info=1 + use_syslog=0 + use_true_regexp_matching=0 + cgi: + template: | + action_url_target=_blank + authorized_for_all_host_commands=* + authorized_for_all_hosts=* + authorized_for_all_service_commands=* + authorized_for_all_services=* + authorized_for_configuration_information=* + authorized_for_system_commands=nagiosadmin + authorized_for_system_information=* + default_statuswrl_layout=4 + enable_page_tour=0 + escape_html_tags=1 + lock_author_names=1 + main_config_file=/opt/nagios/etc/nagios.cfg + navbar_search_for_addresses=1 + navbar_search_for_aliases=1 + notes_url_target=_blank + physical_html_path=/opt/nagios/share + ping_syntax=/bin/ping -n -U -c 5 $HOSTADDRESS$ + refresh_rate=90 + result_limit=100 + show_context_help=0 + url_html_path=/nagios + use_authentication=0 + use_pending_states=1 + use_ssl_authentication=0 + query_es_clauses: null + additionalPlugins: [] +... diff --git a/namespace-config/Chart.yaml b/namespace-config/Chart.yaml new file mode 100644 index 0000000000..cde620369d --- /dev/null +++ b/namespace-config/Chart.yaml @@ -0,0 +1,20 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: v1.0.0 +description: OpenStack-Helm Namespace Config +name: namespace-config +version: 2024.2.0 +home: https://kubernetes.io/docs/concepts/policy/limit-range/ +... diff --git a/namespace-config/templates/limit-range.yaml b/namespace-config/templates/limit-range.yaml new file mode 100644 index 0000000000..ac3f0785ae --- /dev/null +++ b/namespace-config/templates/limit-range.yaml @@ -0,0 +1,20 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +apiVersion: v1 +kind: LimitRange +metadata: + name: {{ printf "%s-%s" .Release.Name "limit-range" }} +spec: +{{ toYaml (dict "limits" .Values.limits) | indent 2 }} diff --git a/namespace-config/values.yaml b/namespace-config/values.yaml new file mode 100644 index 0000000000..62ba156118 --- /dev/null +++ b/namespace-config/values.yaml @@ -0,0 +1,28 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for memcached. +# This is a YAML-formatted file. +# Declare name/value pairs to be passed into your templates. +# name: value + +--- +limits: + - type: Container + default: + cpu: 8 + memory: 8192Mi + defaultRequest: + cpu: 0.1 + memory: 64Mi + +... diff --git a/nfs-provisioner/Chart.yaml b/nfs-provisioner/Chart.yaml new file mode 100644 index 0000000000..356f03bfe7 --- /dev/null +++ b/nfs-provisioner/Chart.yaml @@ -0,0 +1,29 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: v2.2.1 +description: OpenStack-Helm NFS +name: nfs-provisioner +version: 2024.2.0 +home: https://github.com/kubernetes-incubator/external-storage +sources: + - https://github.com/kubernetes-incubator/external-storage + - https://opendev.org/openstack/openstack-helm +maintainers: + - name: OpenStack-Helm Authors +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit + version: ">= 0.1.0" +... diff --git a/nfs-provisioner/templates/configmap-bin.yaml b/nfs-provisioner/templates/configmap-bin.yaml new file mode 100644 index 0000000000..b9450b8c3a --- /dev/null +++ b/nfs-provisioner/templates/configmap-bin.yaml @@ -0,0 +1,25 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_bin }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: nfs-bin +data: + image-repo-sync.sh: | +{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} +{{- end }} diff --git a/nfs-provisioner/templates/deployment.yaml b/nfs-provisioner/templates/deployment.yaml new file mode 100644 index 0000000000..87b2d32a78 --- /dev/null +++ b/nfs-provisioner/templates/deployment.yaml @@ -0,0 +1,208 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.deployment }} +{{- $envAll := . }} + +{{- $serviceAccountName := printf "%s-%s" .Release.Name "nfs-provisioner" }} +{{ tuple $envAll "nfs" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ $serviceAccountName }} +rules: + - apiGroups: + - '' + resources: + - persistentvolumes + verbs: + - get + - list + - watch + - create + - delete + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - get + - list + - watch + - update + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - '' + resources: + - events + verbs: + - list + - watch + - create + - update + - patch + - apiGroups: + - '' + resources: + - services + verbs: + - get + - apiGroups: + - '' + resources: + - endpoints + verbs: + - get + - create + - update + - patch + - apiGroups: + - policy + resources: + - podsecuritypolicies + resourceNames: + - nfs-provisioner + verbs: + - use +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ $serviceAccountName }} + apiGroup: rbac.authorization.k8s.io +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: nfs-provisioner + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "nfs" "provisioner" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + replicas: {{ .Values.pod.replicas.server }} + strategy: + type: Recreate + selector: + matchLabels: +{{ tuple $envAll "nfs" "provisioner" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} + template: + metadata: + labels: +{{ tuple $envAll "nfs" "provisioner" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + spec: + serviceAccountName: {{ $serviceAccountName }} + affinity: +{{ tuple $envAll "nfs" "provisioner" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} + nodeSelector: + {{ .Values.labels.nfs.node_selector_key }}: {{ .Values.labels.nfs.node_selector_value | quote }} + initContainers: +{{ tuple $envAll "nfs" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: nfs-provisioner +{{ tuple $envAll "nfs_provisioner" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + securityContext: + capabilities: + add: + - DAC_READ_SEARCH + - SYS_RESOURCE + ports: + - name: nfs + containerPort: 2049 + - name: nfs-udp + containerPort: 2049 + protocol: UDP + - name: mountd + containerPort: 20048 + - name: mountd-udp + containerPort: 20048 + protocol: UDP + - name: rpcbind + containerPort: 111 + - name: rpcbind-udp + containerPort: 111 + protocol: UDP + - name: port-662 + containerPort: 662 + - name: port-662-udp + containerPort: 662 + protocol: UDP + - name: port-875 + containerPort: 875 + - name: port-875-udp + containerPort: 875 + protocol: UDP + - name: port-32803 + containerPort: 32803 + - name: port-32803-udp + containerPort: 32803 + protocol: UDP + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: SERVICE_NAME + value: {{ tuple "nfs" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + args: + {{ if empty .Values.storageclass.provisioner -}} + - "-provisioner=nfs/{{ .Release.Name }}" + {{- else -}} + - "-provisioner={{ .Values.storageclass.provisioner }}" + {{- end }} + - "-grace-period=10" + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: export-volume + mountPath: /export + volumes: + - name: pod-tmp + emptyDir: {} + - name: export-volume + {{- if eq .Values.storage.type "persistentVolumeClaim" }} + persistentVolumeClaim: + {{ if empty .Values.storage.persistentVolumeClaim.name -}} + claimName: {{ .Release.Name }} + {{- else -}} + claimName: {{ .Values.storage.persistentVolumeClaim.name }} + {{- end }} + {{- else if eq .Values.storage.type "hostPath" }} + hostPath: + path: {{ .Values.storage.hostPath.path }} + {{- end }} +{{- end }} diff --git a/nfs-provisioner/templates/job-image-repo-sync.yaml b/nfs-provisioner/templates/job-image-repo-sync.yaml new file mode 100644 index 0000000000..fa17c3aae1 --- /dev/null +++ b/nfs-provisioner/templates/job-image-repo-sync.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "nfs-provisioner" -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/nfs-provisioner/templates/secret-registry.yaml b/nfs-provisioner/templates/secret-registry.yaml new file mode 100644 index 0000000000..da979b3223 --- /dev/null +++ b/nfs-provisioner/templates/secret-registry.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_registry .Values.endpoints.oci_image_registry.auth.enabled }} +{{ include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) }} +{{- end }} diff --git a/nfs-provisioner/templates/service.yaml b/nfs-provisioner/templates/service.yaml new file mode 100644 index 0000000000..87f294f760 --- /dev/null +++ b/nfs-provisioner/templates/service.yaml @@ -0,0 +1,58 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.service }} +{{- $envAll := . }} +--- +kind: Service +apiVersion: v1 +metadata: + name: {{ tuple "nfs" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + labels: +{{ tuple $envAll "nfs" "provisioner" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + ports: + - name: nfs + port: 2049 + - name: nfs-udp + port: 2049 + protocol: UDP + - name: mountd + port: 20048 + - name: mountd-udp + port: 20048 + protocol: UDP + - name: rpcbind + port: 111 + - name: rpcbind-udp + port: 111 + protocol: UDP + - name: port-662 + port: 662 + - name: port-662-udp + port: 662 + protocol: UDP + - name: port-875 + port: 875 + - name: port-875-udp + port: 875 + protocol: UDP + - name: port-32803 + port: 32803 + - name: port-32803-udp + port: 32803 + protocol: UDP + selector: +{{ tuple $envAll "nfs" "provisioner" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- end }} diff --git a/nfs-provisioner/templates/storage_class.yaml b/nfs-provisioner/templates/storage_class.yaml new file mode 100644 index 0000000000..99614d3d55 --- /dev/null +++ b/nfs-provisioner/templates/storage_class.yaml @@ -0,0 +1,33 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.storage_class }} +{{- $envAll := . }} +--- +kind: StorageClass +apiVersion: storage.k8s.io/v1 +metadata: + {{ if empty .Values.storageclass.name -}} + name: {{ .Release.Name }} + {{- else -}} + name: {{ .Values.storageclass.name }} + {{- end }} +{{ if empty .Values.storageclass.provisioner -}} +provisioner: nfs/{{ .Release.Name }} +{{- else -}} +provisioner: {{ .Values.storageclass.provisioner }} +{{- end }} +parameters: + mountOptions: vers=4.1 +{{- end }} diff --git a/nfs-provisioner/templates/volume_claim.yaml b/nfs-provisioner/templates/volume_claim.yaml new file mode 100644 index 0000000000..755a7590bf --- /dev/null +++ b/nfs-provisioner/templates/volume_claim.yaml @@ -0,0 +1,35 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.volume_claim }} +{{- if eq .Values.storage.type "persistentVolumeClaim" }} +{{- $envAll := . }} +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + {{ if empty .Values.storage.persistentVolumeClaim.name -}} + name: {{ .Release.Name }} + {{- else -}} + name: {{ .Values.storage.persistentVolumeClaim.name }} + {{- end }} +spec: + accessModes: + - {{ .Values.storage.persistentVolumeClaim.access_mode }} + resources: + requests: + storage: {{ .Values.storage.persistentVolumeClaim.size }} + storageClassName: {{ .Values.storage.persistentVolumeClaim.class_name }} +{{- end }} +{{- end }} diff --git a/nfs-provisioner/values.yaml b/nfs-provisioner/values.yaml new file mode 100644 index 0000000000..f7a327ad66 --- /dev/null +++ b/nfs-provisioner/values.yaml @@ -0,0 +1,157 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for NFS. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +--- +pod: + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + weight: + default: 10 + replicas: + # only 1 replica currently supported + server: 1 + resources: + enabled: false + server: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + jobs: + image_repo_sync: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + +images: + tags: + nfs_provisioner: quay.io/kubernetes_incubator/nfs-provisioner:v2.3.0 + dep_check: quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal + image_repo_sync: docker.io/library/docker:17.07.0 + pull_policy: IfNotPresent + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +storage: + type: hostPath + hostPath: + path: /var/lib/openstack-helm/nfs + persistentVolumeClaim: + access_mode: ReadWriteOnce + class_name: general + # NOTE(portdirect): Unless explicity set the PV name will be populated to + # match "{{ .Release.Name }}". + name: null + size: 10Gi + +labels: + nfs: + node_selector_key: openstack-control-plane + node_selector_value: enabled + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +storageclass: + # NOTE(portdirect): Unless explicity set the provisioner name will be generated + # with the format "nfs/{{ .Release.Name }}" + provisioner: null + # NOTE(portdirect): Unless explicity set the PV name will be populated to + # match "{{ .Release.Name }}". + name: null + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - nfs-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + nfs: + services: null + +secrets: + oci_image_registry: + nfs-provisioner: nfs-provisioner-oci-image-registry-key + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + nfs-provisioner: + username: nfs-provisioner + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + nfs: + hosts: + default: nfs-provisioner + host_fqdn_override: + default: null + path: null + scheme: null + port: + nfs: + default: null + +manifests: + configmap_bin: true + deployment: true + job_image_repo_sync: true + secret_registry: true + service: true + storage_class: true + volume_claim: true +... diff --git a/openvswitch/.helmignore b/openvswitch/.helmignore new file mode 100644 index 0000000000..b54c347b85 --- /dev/null +++ b/openvswitch/.helmignore @@ -0,0 +1 @@ +values_overrides diff --git a/openvswitch/Chart.yaml b/openvswitch/Chart.yaml new file mode 100644 index 0000000000..1c9fb94d0a --- /dev/null +++ b/openvswitch/Chart.yaml @@ -0,0 +1,30 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: v1.0.0 +description: OpenStack-Helm OpenVSwitch +name: openvswitch +version: 2024.2.0 +home: http://openvswitch.org +icon: https://www.openstack.org/themes/openstack/images/project-mascots/Neutron/OpenStack_Project_Neutron_vertical.png +sources: + - https://github.com/openvswitch/ovs + - https://opendev.org/openstack/openstack-helm +maintainers: + - name: OpenStack-Helm Authors +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit + version: ">= 0.1.0" +... diff --git a/openvswitch/templates/bin/_openvswitch-db-server.sh.tpl b/openvswitch/templates/bin/_openvswitch-db-server.sh.tpl new file mode 100644 index 0000000000..c3c4845579 --- /dev/null +++ b/openvswitch/templates/bin/_openvswitch-db-server.sh.tpl @@ -0,0 +1,56 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +COMMAND="${@:-start}" + +OVS_DB=/run/openvswitch/conf.db +OVS_SCHEMA=/usr/share/openvswitch/vswitch.ovsschema +OVS_PID=/run/openvswitch/ovsdb-server.pid +OVS_SOCKET=/run/openvswitch/db.sock + +function start () { + mkdir -p "$(dirname ${OVS_DB})" + if [[ ! -e "${OVS_DB}" ]]; then + ovsdb-tool create "${OVS_DB}" + fi + + if [[ "$(ovsdb-tool needs-conversion ${OVS_DB} ${OVS_SCHEMA})" == 'yes' ]]; then + ovsdb-tool convert ${OVS_DB} ${OVS_SCHEMA} + fi + + umask 000 + exec /usr/sbin/ovsdb-server ${OVS_DB} \ + -vconsole:emer \ + -vconsole:err \ + -vconsole:info \ + --pidfile=${OVS_PID} \ + --remote=punix:${OVS_SOCKET} \ + --remote=db:Open_vSwitch,Open_vSwitch,manager_options \ +{{- if .Values.conf.openvswitch_db_server.ptcp_port }} + --remote=ptcp:{{ .Values.conf.openvswitch_db_server.ptcp_port }} \ +{{- end }} + --private-key=db:Open_vSwitch,SSL,private_key \ + --certificate=db:Open_vSwitch,SSL,certificate \ + --bootstrap-ca-cert=db:Open_vSwitch,SSL,ca_cert +} + +function stop () { + PID=$(cat $OVS_PID) + ovs-appctl -T1 -t /run/openvswitch/ovsdb-server.${PID}.ctl exit +} + +$COMMAND diff --git a/openvswitch/templates/bin/_openvswitch-vswitchd-init-modules.sh.tpl b/openvswitch/templates/bin/_openvswitch-vswitchd-init-modules.sh.tpl new file mode 100644 index 0000000000..6e4fdbbfb8 --- /dev/null +++ b/openvswitch/templates/bin/_openvswitch-vswitchd-init-modules.sh.tpl @@ -0,0 +1,26 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +chroot /mnt/host-rootfs modprobe openvswitch +chroot /mnt/host-rootfs modprobe gre +chroot /mnt/host-rootfs modprobe vxlan + +{{- if .Values.conf.ovs_dpdk.enabled }} +{{- if hasKey .Values.conf.ovs_dpdk "driver"}} +chroot /mnt/host-rootfs modprobe {{ .Values.conf.ovs_dpdk.driver | quote }} +{{- end }} +{{- end }} diff --git a/openvswitch/templates/bin/_openvswitch-vswitchd.sh.tpl b/openvswitch/templates/bin/_openvswitch-vswitchd.sh.tpl new file mode 100644 index 0000000000..89f882a321 --- /dev/null +++ b/openvswitch/templates/bin/_openvswitch-vswitchd.sh.tpl @@ -0,0 +1,180 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +COMMAND="${@:-start}" + +OVS_SOCKET=/run/openvswitch/db.sock +OVS_PID=/run/openvswitch/ovs-vswitchd.pid + +# Create vhostuser directory and grant nova user (default UID 42424) access +# permissions. +{{- if .Values.conf.ovs_dpdk.enabled }} +mkdir -p /run/openvswitch/{{ .Values.conf.ovs_dpdk.vhostuser_socket_dir }} +chown {{ .Values.pod.user.nova.uid }}.{{ .Values.pod.user.nova.uid }} /run/openvswitch/{{ .Values.conf.ovs_dpdk.vhostuser_socket_dir }} +chown {{ .Values.pod.user.nova.uid }}.{{ .Values.pod.user.nova.uid }} {{ .Values.conf.ovs_dpdk.hugepages_mountpath }} +{{- end }} + +function start () { + t=0 + while [ ! -e "${OVS_SOCKET}" ] ; do + echo "waiting for ovs socket $sock" + sleep 1 + t=$(($t+1)) + if [ $t -ge 10 ] ; then + echo "no ovs socket, giving up" + exit 1 + fi + done + + ovs-vsctl --db=unix:${OVS_SOCKET} --no-wait show +{{- if .Values.conf.ovs_hw_offload.enabled }} + ovs-vsctl --db=unix:${OVS_SOCKET} --no-wait set Open_vSwitch . other_config:hw-offload={{ .Values.conf.ovs_hw_offload.enabled }} +{{- end }} +{{- if .Values.conf.ovs_other_config.handler_threads }} + ovs-vsctl --db=unix:${OVS_SOCKET} --no-wait set Open_vSwitch . other_config:n-handler-threads={{ .Values.conf.ovs_other_config.handler_threads }} +{{- end }} +{{- if .Values.conf.ovs_other_config.revalidator_threads }} + ovs-vsctl --db=unix:${OVS_SOCKET} --no-wait set Open_vSwitch . other_config:n-revalidator-threads={{ .Values.conf.ovs_other_config.revalidator_threads }} +{{- end }} + +{{- if .Values.conf.ovs_dpdk.enabled }} + ovs-vsctl --db=unix:${OVS_SOCKET} --no-wait set Open_vSwitch . other_config:dpdk-hugepage-dir={{ .Values.conf.ovs_dpdk.hugepages_mountpath | quote }} + ovs-vsctl --db=unix:${OVS_SOCKET} --no-wait set Open_vSwitch . other_config:dpdk-socket-mem={{ .Values.conf.ovs_dpdk.socket_memory | quote }} + +{{- if .Values.conf.ovs_dpdk.mem_channels }} + ovs-vsctl --db=unix:${OVS_SOCKET} --no-wait set Open_vSwitch . other_config:dpdk-mem-channels={{ .Values.conf.ovs_dpdk.mem_channels | quote }} +{{- end }} + +{{- if hasKey .Values.conf.ovs_dpdk "pmd_cpu_mask" }} + ovs-vsctl --db=unix:${OVS_SOCKET} --no-wait set Open_vSwitch . other_config:pmd-cpu-mask={{ .Values.conf.ovs_dpdk.pmd_cpu_mask | quote }} + PMD_CPU_MASK={{ .Values.conf.ovs_dpdk.pmd_cpu_mask | quote }} +{{- end }} + +{{- if hasKey .Values.conf.ovs_dpdk "lcore_mask" }} + ovs-vsctl --db=unix:${OVS_SOCKET} --no-wait set Open_vSwitch . other_config:dpdk-lcore-mask={{ .Values.conf.ovs_dpdk.lcore_mask | quote }} + LCORE_MASK={{ .Values.conf.ovs_dpdk.lcore_mask | quote }} +{{- end }} + +{{- if hasKey .Values.conf.ovs_dpdk "vhost_iommu_support" }} + ovs-vsctl --db=unix:${OVS_SOCKET} --no-wait set Open_vSwitch . other_config:vhost-iommu-support={{ .Values.conf.ovs_dpdk.vhost_iommu_support }} +{{- end }} + + ovs-vsctl --db=unix:${OVS_SOCKET} --no-wait set Open_vSwitch . other_config:vhost-sock-dir={{ .Values.conf.ovs_dpdk.vhostuser_socket_dir | quote }} + ovs-vsctl --db=unix:${OVS_SOCKET} --no-wait set Open_vSwitch . other_config:dpdk-init=true + + # No need to create the cgroup if lcore_mask or pmd_cpu_mask is not set. + if [[ -n ${PMD_CPU_MASK} || -n ${LCORE_MASK} ]]; then + if [ "$(stat -fc %T /sys/fs/cgroup/)" = "cgroup2fs" ]; then + # Setup Cgroups to use when breaking out of Kubernetes defined groups + mkdir -p /sys/fs/cgroup/osh-openvswitch + target_mems="/sys/fs/cgroup/osh-openvswitch/cpuset.mems" + target_cpus="/sys/fs/cgroup/osh-openvswitch/cpuset.cpus" + touch $target_mems + touch $target_cpus + + # Ensure the write target for the for cpuset.mem for the pod exists + if [[ -f "$target_mems" && -f "$target_cpus" ]]; then + # Write cpuset.mem and cpuset.cpus for new cgroup and add current task to new cgroup + cat /sys/fs/cgroup/cpuset.mems.effective > "$target_mems" + cat /sys/fs/cgroup/cpuset.cpus.effective > "$target_cpus" + echo $$ > /sys/fs/cgroup/osh-openvswitch/cgroup.procs + else + echo "ERROR: Could not find write target for either cpuset.mems: $target_mems or cpuset.cpus: $target_cpus" + fi + else + # Setup Cgroups to use when breaking out of Kubernetes defined groups + mkdir -p /sys/fs/cgroup/cpuset/osh-openvswitch + target_mems="/sys/fs/cgroup/cpuset/osh-openvswitch/cpuset.mems" + target_cpus="/sys/fs/cgroup/cpuset/osh-openvswitch/cpuset.cpus" + + # Ensure the write target for the for cpuset.mem for the pod exists + if [[ -f "$target_mems" && -f "$target_cpus" ]]; then + # Write cpuset.mem and cpuset.cpus for new cgroup and add current task to new cgroup + cat /sys/fs/cgroup/cpuset/cpuset.mems > "$target_mems" + cat /sys/fs/cgroup/cpuset/cpuset.cpus > "$target_cpus" + echo $$ > /sys/fs/cgroup/cpuset/osh-openvswitch/tasks + else + echo "ERROR: Could not find write target for either cpuset.mems: $target_mems or cpuset.cpus: $target_cpus" + fi + fi + fi +{{- end }} + + exec /usr/sbin/ovs-vswitchd unix:${OVS_SOCKET} \ + -vconsole:emer \ + -vconsole:err \ + -vconsole:info \ + --pidfile=${OVS_PID} \ + {{- if .Values.conf.ovs_user_name }} + --user="{{ .Values.conf.ovs_user_name }}" \ + {{- end }} + --mlockall +} + +function stop () { + PID=$(cat $OVS_PID) + ovs-appctl -T1 -t /run/openvswitch/ovs-vswitchd.${PID}.ctl exit +} + +find_latest_ctl_file() { + latest_file="" + latest_file=$(ls -lt /run/openvswitch/*.ctl | awk 'NR==1 {if ($3 == "{{ .Values.conf.poststart.rootUser }}") print $NF}') + + echo "$latest_file" +} + +function poststart () { + # This enables the usage of 'ovs-appctl' from neutron-ovs-agent pod. + + # Wait for potential new ctl file before continuing + timeout={{ .Values.conf.poststart.timeout }} + start_time=$(date +%s) + while true; do + latest_ctl_file=$(find_latest_ctl_file) + if [ -n "$latest_ctl_file" ]; then + break + fi + current_time=$(date +%s) + if (( current_time - start_time >= timeout )); then + break + fi + sleep 1 + done + + until [ -f $OVS_PID ] + do + echo "Waiting for file $OVS_PID" + sleep 1 + done + + PID=$(cat $OVS_PID) + OVS_CTL=/run/openvswitch/ovs-vswitchd.${PID}.ctl + + until [ -S $OVS_CTL ] + do + echo "Waiting for file $OVS_CTL" + sleep 1 + done + chown {{ .Values.pod.user.nova.uid }}.{{ .Values.pod.user.nova.uid }} ${OVS_CTL} + +{{- if .Values.conf.poststart.extraCommand }} +{{ .Values.conf.poststart.extraCommand | indent 2 }} +{{- end }} + +} + +$COMMAND diff --git a/openvswitch/templates/configmap-bin.yaml b/openvswitch/templates/configmap-bin.yaml new file mode 100644 index 0000000000..f6e8dc53b3 --- /dev/null +++ b/openvswitch/templates/configmap-bin.yaml @@ -0,0 +1,33 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_bin }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: openvswitch-bin +data: +{{- if .Values.images.local_registry.active }} + image-repo-sync.sh: | +{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} +{{- end }} + openvswitch-db-server.sh: | +{{ tuple "bin/_openvswitch-db-server.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + openvswitch-vswitchd.sh: | +{{ tuple "bin/_openvswitch-vswitchd.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + openvswitch-vswitchd-init-modules.sh: | +{{ tuple "bin/_openvswitch-vswitchd-init-modules.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} diff --git a/openvswitch/templates/daemonset.yaml b/openvswitch/templates/daemonset.yaml new file mode 100644 index 0000000000..a6c7527b5e --- /dev/null +++ b/openvswitch/templates/daemonset.yaml @@ -0,0 +1,274 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "ovsdblivenessProbeTemplate" }} +exec: + command: + - /usr/bin/ovs-vsctl + - show +{{- end }} + +{{- define "ovsdbreadinessProbeTemplate" }} +exec: + command: + - /usr/bin/ovs-vsctl + - list + - Open_Vswitch +{{- end }} + +{{- define "ovsvswitchlivenessProbeTemplate" }} +exec: + command: +{{- if .Values.pod.probes.ovs.ovs_vswitch.liveness.exec }} +{{ .Values.pod.probes.ovs.ovs_vswitch.liveness.exec | toYaml | indent 4 }} +{{- else }} + - /usr/bin/ovs-appctl + - bond/list +{{- end }} +{{- end }} + +{{- define "ovsvswitchreadinessProbeTemplate" }} +exec: + command: +{{- if .Values.pod.probes.ovs.ovs_vswitch.readiness.exec }} +{{ .Values.pod.probes.ovs.ovs_vswitch.readiness.exec | toYaml | indent 4 }} +{{- else if not .Values.conf.ovs_dpdk.enabled }} + - /bin/bash + - -c + - '/usr/bin/ovs-vsctl show' +{{- else }} + - /bin/bash + - -c + - '/usr/bin/ovs-vsctl show && ! /usr/bin/ovs-vsctl list Open_vSwitch | grep -q dpdk_initialized.*false' +{{- end }} +{{- end }} + +{{- if .Values.manifests.daemonset }} +{{- $envAll := . }} + +{{- $serviceAccountName := "openvswitch-server" }} +{{ tuple $envAll "ovs" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: openvswitch + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "openvswitch" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + selector: + matchLabels: +{{ tuple $envAll "openvswitch" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} +{{ tuple $envAll "ovs" | include "helm-toolkit.snippets.kubernetes_upgrades_daemonset" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "openvswitch" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} +{{ dict "envAll" $envAll "podName" "openvswitch" "containerNames" (list "openvswitch-db" "openvswitch-db-perms" "openvswitch-vswitchd" "openvswitch-vswitchd-modules" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: + shareProcessNamespace: true + serviceAccountName: {{ $serviceAccountName }} +{{ dict "envAll" $envAll "application" "ovs" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + nodeSelector: + {{ .Values.labels.ovs.node_selector_key }}: {{ .Values.labels.ovs.node_selector_value }} +{{ if $envAll.Values.pod.tolerations.openvswitch.enabled }} +{{ tuple $envAll "openvswitch" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }} +{{ end }} + dnsPolicy: {{ .Values.pod.dns_policy }} + hostNetwork: true + initContainers: +{{ tuple $envAll "ovs" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + - name: openvswitch-db-perms +{{ tuple $envAll "openvswitch_db_server" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ dict "envAll" $envAll "application" "ovs" "container" "perms" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.ovs.db | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + command: + - chown + - -R + - {{ $envAll.Values.pod.security_context.ovs.container.server.runAsUser | quote }} + - /run/openvswitch + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: run-openvswitch + mountPath: /run/openvswitch + - name: openvswitch-vswitchd-modules +{{ tuple $envAll "openvswitch_vswitchd" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ dict "envAll" $envAll "application" "ovs" "container" "modules" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/openvswitch-vswitchd-init-modules.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: openvswitch-bin + mountPath: /tmp/openvswitch-vswitchd-init-modules.sh + subPath: openvswitch-vswitchd-init-modules.sh + readOnly: true + - name: host-rootfs + mountPath: /mnt/host-rootfs + mountPropagation: HostToContainer + readOnly: true + containers: + - name: openvswitch-db +{{ tuple $envAll "openvswitch_db_server" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ dict "envAll" $envAll "application" "ovs" "container" "server" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.ovs.db | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "component" "ovs" "container" "ovs_db" "type" "liveness" "probeTemplate" (include "ovsdblivenessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }} +{{ dict "envAll" $envAll "component" "ovs" "container" "ovs_db" "type" "readiness" "probeTemplate" (include "ovsdbreadinessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }} + command: + - /tmp/openvswitch-db-server.sh + - start + lifecycle: + preStop: + exec: + command: + - /tmp/openvswitch-db-server.sh + - stop + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: openvswitch-bin + mountPath: /tmp/openvswitch-db-server.sh + subPath: openvswitch-db-server.sh + readOnly: true + - name: run + mountPath: /run + - name: openvswitch-vswitchd +{{/* Run the container in priviledged mode due to the need for root +permissions when we specify --user to run in non-root. */}} +{{- $_ := set $envAll.Values.pod.security_context.ovs.container.vswitchd "privileged" true -}} +{{- if .Values.conf.ovs_dpdk.enabled }} +{{/* Limiting CPU cores would severely affect packet throughput +It should be handled through lcore and pmd core masks. */}} +{{- if .Values.pod.resources.enabled }} +{{ $_ := unset $envAll.Values.pod.resources.ovs.vswitchd.requests "cpu" }} +{{ $_ := unset $envAll.Values.pod.resources.ovs.vswitchd.limits "cpu" }} +{{- end }} +{{- end }} +{{ tuple $envAll "openvswitch_vswitchd" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ dict "envAll" $envAll "application" "ovs" "container" "vswitchd" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.ovs.vswitchd | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + # ensures this container can speak to the ovs database + # successfully before its marked as ready +{{ dict "envAll" $envAll "component" "ovs" "container" "ovs_vswitch" "type" "liveness" "probeTemplate" (include "ovsvswitchlivenessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }} +{{ dict "envAll" $envAll "component" "ovs" "container" "ovs_vswitch" "type" "readiness" "probeTemplate" (include "ovsvswitchreadinessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }} +{{- if .Values.pod.tini.enabled }} + command: + - /tini + - -s + - -- + args: + - /tmp/openvswitch-vswitchd.sh + - start +{{- else }} + command: + - /tmp/openvswitch-vswitchd.sh + - start +{{- end }} + lifecycle: + postStart: + exec: + command: + - /tmp/openvswitch-vswitchd.sh + - poststart + preStop: + exec: + command: + - /tmp/openvswitch-vswitchd.sh + - stop + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: openvswitch-bin + mountPath: /tmp/openvswitch-vswitchd.sh + subPath: openvswitch-vswitchd.sh + readOnly: true + - name: run + mountPath: /run +{{- if .Values.conf.ovs_dpdk.enabled }} + - name: hugepages + mountPath: {{ .Values.conf.ovs_dpdk.hugepages_mountpath | quote }} + - name: pci-devices + mountPath: /sys/bus/pci/devices + - name: huge-pages-kernel + mountPath: /sys/kernel/mm/hugepages + - name: node-devices + mountPath: /sys/devices/system/node + - name: modules + mountPath: /lib/modules + - name: devs + mountPath: /dev + - name: pci-drivers + mountPath: /sys/bus/pci/drivers + - name: cgroup + mountPath: /sys/fs/cgroup +{{- end }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: openvswitch-bin + configMap: + name: openvswitch-bin + defaultMode: 0555 + - name: run + hostPath: + path: /run + type: Directory + - name: run-openvswitch + hostPath: + path: /run/openvswitch + type: DirectoryOrCreate + - name: host-rootfs + hostPath: + path: / + type: Directory +{{- if .Values.conf.ovs_dpdk.enabled }} + - name: devs + hostPath: + path: /dev + type: Directory + - name: pci-devices + hostPath: + path: /sys/bus/pci/devices + type: Directory + - name: huge-pages-kernel + hostPath: + path: /sys/kernel/mm/hugepages + type: Directory + - name: node-devices + hostPath: + path: /sys/devices/system/node + type: Directory + - name: modules + hostPath: + path: /lib/modules + type: Directory + - name: pci-drivers + hostPath: + path: /sys/bus/pci/drivers + type: Directory + - name: hugepages + hostPath: + path: {{ .Values.conf.ovs_dpdk.hugepages_mountpath | quote }} + type: Directory + - name: cgroup + hostPath: + path: /sys/fs/cgroup +{{- end }} +{{- end }} \ No newline at end of file diff --git a/openvswitch/templates/job-image-repo-sync.yaml b/openvswitch/templates/job-image-repo-sync.yaml new file mode 100644 index 0000000000..765061c320 --- /dev/null +++ b/openvswitch/templates/job-image-repo-sync.yaml @@ -0,0 +1,21 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "openvswitch" -}} +{{- if .Values.pod.tolerations.openvswitch.enabled -}} +{{- $_ := set $imageRepoSyncJob "tolerationsEnabled" true -}} +{{- end -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/openvswitch/templates/network-policy.yaml b/openvswitch/templates/network-policy.yaml new file mode 100644 index 0000000000..751e0e0c10 --- /dev/null +++ b/openvswitch/templates/network-policy.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.network_policy -}} +{{- $netpol_opts := dict "envAll" . "name" "application" "label" "openvswitch" -}} +{{ $netpol_opts | include "helm-toolkit.manifests.kubernetes_network_policy" }} +{{- end -}} diff --git a/openvswitch/templates/secret-registry.yaml b/openvswitch/templates/secret-registry.yaml new file mode 100644 index 0000000000..da979b3223 --- /dev/null +++ b/openvswitch/templates/secret-registry.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_registry .Values.endpoints.oci_image_registry.auth.enabled }} +{{ include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) }} +{{- end }} diff --git a/openvswitch/values.yaml b/openvswitch/values.yaml new file mode 100644 index 0000000000..89aeb88f66 --- /dev/null +++ b/openvswitch/values.yaml @@ -0,0 +1,250 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for openvswitch. +# This is a YAML-formatted file. +# Declare name/value pairs to be passed into your templates. +# name: value + +--- +release_group: null + +images: + tags: + openvswitch_db_server: docker.io/openstackhelm/openvswitch:latest-ubuntu_focal + openvswitch_vswitchd: docker.io/openstackhelm/openvswitch:latest-ubuntu_focal + dep_check: quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal + image_repo_sync: docker.io/library/docker:17.07.0 + pull_policy: "IfNotPresent" + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +labels: + ovs: + node_selector_key: openvswitch + node_selector_value: enabled + +pod: + tini: + enabled: true + tolerations: + openvswitch: + enabled: false + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + probes: + ovs: + ovs_db: + liveness: + enabled: true + params: + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 5 + readiness: + enabled: true + params: + initialDelaySeconds: 90 + periodSeconds: 30 + timeoutSeconds: 5 + ovs_vswitch: + liveness: + enabled: true + params: + initialDelaySeconds: 60 + periodSeconds: 30 + timeoutSeconds: 5 + readiness: + enabled: true + params: + failureThreshold: 3 + periodSeconds: 10 + timeoutSeconds: 1 + security_context: + ovs: + pod: + runAsUser: 42424 + container: + perms: + runAsUser: 0 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + server: + runAsUser: 42424 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + modules: + runAsUser: 0 + capabilities: + add: + - SYS_MODULE + - SYS_CHROOT + readOnlyRootFilesystem: true + vswitchd: + runAsUser: 0 + capabilities: + add: + - NET_ADMIN + readOnlyRootFilesystem: true + dns_policy: "ClusterFirstWithHostNet" + lifecycle: + upgrades: + daemonsets: + pod_replacement_strategy: RollingUpdate + ovs: + enabled: true + min_ready_seconds: 0 + max_unavailable: 1 + resources: + enabled: false + ovs: + db: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + vswitchd: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + # set resources to enabled and specify one of the following when using dpdk + # hugepages-1Gi: "1Gi" + # hugepages-2Mi: "512Mi" + jobs: + image_repo_sync: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + user: + nova: + uid: 42424 + +secrets: + oci_image_registry: + openvswitch: openvswitch-oci-image-registry-key + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + openvswitch: + username: openvswitch + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + +network_policy: + openvswitch: + ingress: + - {} + egress: + - {} + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - openvswitch-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + ovs: null + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + +manifests: + configmap_bin: true + daemonset: true + daemonset_ovs_vswitchd: true + job_image_repo_sync: true + network_policy: false + secret_registry: true + +conf: + poststart: + timeout: 5 + rootUser: "root" + extraCommand: null + openvswitch_db_server: + ptcp_port: null + ovs_other_config: + handler_threads: null + revalidator_threads: null + ovs_hw_offload: + enabled: false + ovs_dpdk: + enabled: false + ## Mandatory parameters. Please uncomment when enabling DPDK + # socket_memory: 1024 + # hugepages_mountpath: /dev/hugepages + # vhostuser_socket_dir: vhostuser + # + ## Optional hardware specific parameters: modify to match NUMA topology + # mem_channels: 4 + # lcore_mask: 0x1 + # pmd_cpu_mask: 0x4 + # + ## Optional driver to use. Driver name should be the same as the one + ## specified in the ovs_dpdk section in the Neutron values and vice versa + # driver: vfio-pci + # + ## Optional security feature + # vHost IOMMU feature restricts the vhost memory that a virtio device + # access, available with DPDK v17.11 + # vhost_iommu_support: true + ## OVS supports run in non-root for both OVS and OVS DPDK mode, the user + # for OVS need to be added to container image with user id 42424. + # useradd -u 42424 openvswitch; groupmod -g 42424 openvswitch + # + # Leave empty to run as user that invokes the command (default: root) + ovs_user_name: "openvswitch:openvswitch" +... diff --git a/ovn/.helmignore b/ovn/.helmignore new file mode 100644 index 0000000000..b54c347b85 --- /dev/null +++ b/ovn/.helmignore @@ -0,0 +1 @@ +values_overrides diff --git a/ovn/Chart.yaml b/ovn/Chart.yaml new file mode 100644 index 0000000000..a9e426d500 --- /dev/null +++ b/ovn/Chart.yaml @@ -0,0 +1,30 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: v23.3.0 +description: OpenStack-Helm OVN +name: ovn +version: 2024.2.0 +home: https://www.ovn.org +icon: https://www.ovn.org/images/ovn-logo.png +sources: + - https://github.com/ovn-org/ovn + - https://opendev.org/openstack/openstack-helm +maintainers: + - name: OpenStack-Helm Authors +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit + version: ">= 0.1.0" +... diff --git a/ovn/templates/bin/_ovn-controller-init.sh.tpl b/ovn/templates/bin/_ovn-controller-init.sh.tpl new file mode 100644 index 0000000000..357c069da4 --- /dev/null +++ b/ovn/templates/bin/_ovn-controller-init.sh.tpl @@ -0,0 +1,179 @@ +#!/bin/bash -xe + +# Copyright 2023 VEXXHOST, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +ANNOTATION_KEY="openstack-helm-infra/ovn-system-id" + +function get_ip_address_from_interface { + local interface=$1 + local ip=$(ip -4 -o addr s "${interface}" | awk '{ print $4; exit }' | awk -F '/' 'NR==1 {print $1}') + if [ -z "${ip}" ] ; then + exit 1 + fi + echo ${ip} +} + +function get_ip_prefix_from_interface { + local interface=$1 + local prefix=$(ip -4 -o addr s "${interface}" | awk '{ print $4; exit }' | awk -F '/' 'NR==1 {print $2}') + if [ -z "${prefix}" ] ; then + exit 1 + fi + echo ${prefix} +} + +function migrate_ip_from_nic { + src_nic=$1 + bridge_name=$2 + + # Enabling explicit error handling: We must avoid to lose the IP + # address in the migration process. Hence, on every error, we + # attempt to assign the IP back to the original NIC and exit. + set +e + + ip=$(get_ip_address_from_interface ${src_nic}) + prefix=$(get_ip_prefix_from_interface ${src_nic}) + + bridge_ip=$(get_ip_address_from_interface "${bridge_name}") + bridge_prefix=$(get_ip_prefix_from_interface "${bridge_name}") + + ip link set ${bridge_name} up + + if [[ -n "${ip}" && -n "${prefix}" ]]; then + ip addr flush dev ${src_nic} + if [ $? -ne 0 ] ; then + ip addr add ${ip}/${prefix} dev ${src_nic} + echo "Error while flushing IP from ${src_nic}." + exit 1 + fi + + ip addr add ${ip}/${prefix} dev "${bridge_name}" + if [ $? -ne 0 ] ; then + echo "Error assigning IP to bridge "${bridge_name}"." + ip addr add ${ip}/${prefix} dev ${src_nic} + exit 1 + fi + elif [[ -n "${bridge_ip}" && -n "${bridge_prefix}" ]]; then + echo "Bridge '${bridge_name}' already has IP assigned. Keeping the same:: IP:[${bridge_ip}]; Prefix:[${bridge_prefix}]..." + elif [[ -z "${bridge_ip}" && -z "${ip}" ]]; then + echo "Interface and bridge have no ips configured. Leaving as is." + else + echo "Interface ${src_nic} has invalid IP address. IP:[${ip}]; Prefix:[${prefix}]..." + exit 1 + fi + + set -e +} + +function get_current_system_id { + ovs-vsctl --if-exists get Open_vSwitch . external_ids:system-id | tr -d '"' +} + +function get_stored_system_id { + kubectl get node "$NODE_NAME" -o "jsonpath={.metadata.annotations.openstack-helm-infra/ovn-system-id}" +} + +function store_system_id() { + local system_id=$1 + kubectl annotate node "$NODE_NAME" "$ANNOTATION_KEY=$system_id" +} + +# Detect tunnel interface +tunnel_interface="{{- .Values.network.interface.tunnel -}}" +if [ -z "${tunnel_interface}" ] ; then + # search for interface with tunnel network routing + tunnel_network_cidr="{{- .Values.network.interface.tunnel_network_cidr -}}" + if [ -z "${tunnel_network_cidr}" ] ; then + tunnel_network_cidr="0/0" + fi + # If there is not tunnel network gateway, exit + tunnel_interface=$(ip -4 route list ${tunnel_network_cidr} | awk -F 'dev' '{ print $2; exit }' \ + | awk '{ print $1 }') || exit 1 +fi +ovs-vsctl set open . external_ids:ovn-encap-ip="$(get_ip_address_from_interface ${tunnel_interface})" + +# Get the stored system-id from the Kubernetes node annotation +stored_system_id=$(get_stored_system_id) + +# Get the current system-id set in OVS +current_system_id=$(get_current_system_id) + +if [ -n "$stored_system_id" ] && [ "$stored_system_id" != "$current_system_id" ]; then + # If the annotation exists and does not match the current system-id, set the system-id to the stored one + ovs-vsctl set Open_vSwitch . external_ids:system-id="$stored_system_id" +elif [ -z "$current_system_id" ]; then + # If no current system-id is set, generate a new one + current_system_id=$(uuidgen) + ovs-vsctl set Open_vSwitch . external_ids:system-id="$current_system_id" + # Store the new system-id in the Kubernetes node annotation + store_system_id "$current_system_id" +elif [ -z "$stored_system_id" ]; then + # If there is no stored system-id, store the current one + store_system_id "$current_system_id" +fi + +# Configure OVN remote +{{- if empty .Values.conf.ovn_remote -}} +{{- $sb_svc_name := "ovn-ovsdb-sb" -}} +{{- $sb_svc := (tuple $sb_svc_name "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup") -}} +{{- $sb_port := (tuple "ovn-ovsdb-sb" "internal" "ovsdb" . | include "helm-toolkit.endpoints.endpoint_port_lookup") -}} +{{- $sb_service_list := list -}} +{{- range $i := until (.Values.pod.replicas.ovn_ovsdb_sb | int) -}} + {{- $sb_service_list = printf "tcp:%s-%d.%s:%s" $sb_svc_name $i $sb_svc $sb_port | append $sb_service_list -}} +{{- end }} + +ovs-vsctl set open . external-ids:ovn-remote="{{ include "helm-toolkit.utils.joinListWithComma" $sb_service_list }}" +{{- else -}} +ovs-vsctl set open . external-ids:ovn-remote="{{ .Values.conf.ovn_remote }}" +{{- end }} + +# Configure OVN values +ovs-vsctl set open . external-ids:rundir="/var/run/openvswitch" +ovs-vsctl set open . external-ids:ovn-encap-type="{{ .Values.conf.ovn_encap_type }}" +ovs-vsctl set open . external-ids:ovn-bridge="{{ .Values.conf.ovn_bridge }}" +ovs-vsctl set open . external-ids:ovn-bridge-mappings="{{ .Values.conf.ovn_bridge_mappings }}" + +GW_ENABLED=$(cat /tmp/gw-enabled/gw-enabled) +if [[ ${GW_ENABLED} == {{ .Values.labels.ovn_controller_gw.node_selector_value }} ]]; then + ovs-vsctl set open . external-ids:ovn-cms-options={{ .Values.conf.ovn_cms_options_gw_enabled }} +else + ovs-vsctl set open . external-ids:ovn-cms-options={{ .Values.conf.ovn_cms_options }} +fi + +{{ if .Values.conf.ovn_bridge_datapath_type -}} +ovs-vsctl set open . external-ids:ovn-bridge-datapath-type="{{ .Values.conf.ovn_bridge_datapath_type }}" +{{- end }} + +# Configure hostname +{{- if .Values.pod.use_fqdn.compute }} + ovs-vsctl set open . external-ids:hostname="$(hostname -f)" +{{- else }} + ovs-vsctl set open . external-ids:hostname="$(hostname)" +{{- end }} + +# Create bridges and create ports +# handle any bridge mappings +# /tmp/auto_bridge_add is one line json file: {"br-ex1":"eth1","br-ex2":"eth2"} +for bmap in `sed 's/[{}"]//g' /tmp/auto_bridge_add | tr "," "\n"` +do + bridge=${bmap%:*} + iface=${bmap#*:} + ovs-vsctl --may-exist add-br $bridge -- set bridge $bridge protocols=OpenFlow13 + if [ -n "$iface" ] && [ "$iface" != "null" ] && ( ip link show $iface 1>/dev/null 2>&1 ); + then + ovs-vsctl --may-exist add-port $bridge $iface + migrate_ip_from_nic $iface $bridge + fi +done diff --git a/ovn/templates/bin/_ovn-network-logging-parser.sh.tpl b/ovn/templates/bin/_ovn-network-logging-parser.sh.tpl new file mode 100644 index 0000000000..06eaaa7f7e --- /dev/null +++ b/ovn/templates/bin/_ovn-network-logging-parser.sh.tpl @@ -0,0 +1,28 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +COMMAND="${@:-start}" + +function start () { + exec uwsgi --ini /etc/neutron/neutron-ovn-network-logging-parser-uwsgi.ini +} + +function stop () { + kill -TERM 1 +} + +$COMMAND diff --git a/ovn/templates/clusterrole-controller.yaml b/ovn/templates/clusterrole-controller.yaml new file mode 100644 index 0000000000..bf2cc23fbf --- /dev/null +++ b/ovn/templates/clusterrole-controller.yaml @@ -0,0 +1,28 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: ovn-controller +rules: +- apiGroups: + - "" + resources: + - nodes + verbs: + - get + - patch + - list diff --git a/ovn/templates/clusterrolebinding-controller.yaml b/ovn/templates/clusterrolebinding-controller.yaml new file mode 100644 index 0000000000..2be1b553bd --- /dev/null +++ b/ovn/templates/clusterrolebinding-controller.yaml @@ -0,0 +1,30 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: ovn-controller +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: ovn-controller +subjects: +- kind: ServiceAccount + name: ovn-controller + namespace: {{ .Release.Namespace }} +- kind: ServiceAccount + name: ovn-controller-gw + namespace: {{ .Release.Namespace }} diff --git a/ovn/templates/configmap-bin.yaml b/ovn/templates/configmap-bin.yaml new file mode 100644 index 0000000000..e61f356e26 --- /dev/null +++ b/ovn/templates/configmap-bin.yaml @@ -0,0 +1,38 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "ovn.configmap.bin" }} +{{- $configMapName := index . 0 }} +{{- $envAll := index . 1 }} +{{- with $envAll }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ $configMapName }} +data: +{{- if .Values.images.local_registry.active }} + image-repo-sync.sh: | +{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} +{{- end }} + ovn-controller-init.sh: | +{{ tuple "bin/_ovn-controller-init.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + ovn-network-logging-parser.sh: | +{{ tuple "bin/_ovn-network-logging-parser.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} +{{- end }} + +{{- if .Values.manifests.configmap_bin }} +{{- list "ovn-bin" . | include "ovn.configmap.bin" }} +{{- end }} diff --git a/ovn/templates/configmap-etc.yaml b/ovn/templates/configmap-etc.yaml new file mode 100644 index 0000000000..0d221f1973 --- /dev/null +++ b/ovn/templates/configmap-etc.yaml @@ -0,0 +1,40 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "ovn.configmap.etc" }} +{{- $configMapName := index . 0 }} +{{- $envAll := index . 1 }} +{{- with $envAll }} + +{{- if empty (index .Values.conf.ovn_network_logging_parser_uwsgi.uwsgi "http-socket") -}} +{{- $http_socket_port := tuple "ovn_logging_parser" "service" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | toString }} +{{- $http_socket := printf "0.0.0.0:%s" $http_socket_port }} +{{- $_ := set .Values.conf.ovn_network_logging_parser_uwsgi.uwsgi "http-socket" $http_socket -}} +{{- end -}} + +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $configMapName }} +type: Opaque +data: + auto_bridge_add: {{ toJson $envAll.Values.conf.auto_bridge_add | b64enc }} + neutron-ovn-network-logging-parser-uwsgi.ini: {{ include "helm-toolkit.utils.to_oslo_conf" .Values.conf.ovn_network_logging_parser_uwsgi | b64enc }} +{{- end }} +{{- end }} + +{{- if .Values.manifests.configmap_etc }} +{{- list "ovn-etc" . | include "ovn.configmap.etc" }} +{{- end }} diff --git a/ovn/templates/daemonset-controller.yaml b/ovn/templates/daemonset-controller.yaml new file mode 100644 index 0000000000..c1122262f0 --- /dev/null +++ b/ovn/templates/daemonset-controller.yaml @@ -0,0 +1,260 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "controllerReadinessProbeTemplate" }} +exec: + command: + - /usr/bin/ovn-kube-util + - readiness-probe + - -t + - ovn-controller +{{- end }} + +{{- define "ovn.daemonset" }} +{{- $daemonset := index . 0 }} +{{- $configMapName := index . 1 }} +{{- $serviceAccountName := index . 2 }} +{{- $envAll := index . 3 }} +{{- with $envAll }} + +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: ovn-controller + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + labels: +{{ tuple $envAll "ovn" "ovn-controller" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + selector: + matchLabels: +{{ tuple $envAll "ovn" "ovn-controller" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} + template: + metadata: + labels: +{{ tuple $envAll "ovn" "ovn-controller" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} + spec: + serviceAccountName: {{ $serviceAccountName }} + hostNetwork: true + hostPID: true + hostIPC: true + dnsPolicy: {{ .Values.pod.dns_policy }} + nodeSelector: + {{ .Values.labels.ovn_controller.node_selector_key }}: {{ .Values.labels.ovn_controller.node_selector_value }} + initContainers: +{{- tuple $envAll "ovn_controller" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + - name: get-gw-enabled +{{ tuple $envAll "ovn_controller_kubectl" | include "helm-toolkit.snippets.image" | indent 10 }} + command: + - /bin/bash + - -c + - | + kubectl get node ${NODENAME} -o jsonpath='{.metadata.labels.{{ .Values.labels.ovn_controller_gw.node_selector_key }}}' > /tmp/gw-enabled/gw-enabled + env: + - name: NODENAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: gw-enabled + mountPath: /tmp/gw-enabled + readOnly: false + - name: controller-init +{{ dict "envAll" $envAll "application" "ovn_controller" "container" "controller_init" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} +{{ tuple $envAll "ovn_controller" | include "helm-toolkit.snippets.image" | indent 10 }} + command: + - /tmp/ovn-controller-init.sh + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: ovn-bin + mountPath: /tmp/ovn-controller-init.sh + subPath: ovn-controller-init.sh + readOnly: true + - name: run-openvswitch + mountPath: /run/openvswitch + - name: ovn-etc + mountPath: /tmp/auto_bridge_add + subPath: auto_bridge_add + readOnly: true + - name: gw-enabled + mountPath: /tmp/gw-enabled + readOnly: true + containers: + - name: controller +{{ tuple $envAll "ovn_controller" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.ovn_controller | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "ovn_controller" "container" "controller" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /root/ovnkube.sh + - ovn-controller +{{ dict "envAll" . "component" "ovn_controller" "container" "controller" "type" "readiness" "probeTemplate" (include "controllerReadinessProbeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }} + env: + - name: OVS_USER_ID + value: {{ .Values.conf.ovs_user_name }} + - name: OVN_DAEMONSET_VERSION + value: "3" + - name: OVN_LOGLEVEL_CONTROLLER + value: "-vconsole:info -vfile:info" + - name: OVN_KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OVN_KUBERNETES_NB_STATEFULSET + value: ovn-ovsdb-nb + - name: OVN_KUBERNETES_SB_STATEFULSET + value: ovn-ovsdb-sb + - name: OVN_SSL_ENABLE + value: "no" + volumeMounts: + - name: run-openvswitch + mountPath: /run/openvswitch + - name: logs + mountPath: /var/log/ovn + - name: run-openvswitch + mountPath: /run/ovn + {{- if .Values.pod.sidecars.vector }} + - name: vector +{{ tuple $envAll "vector" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.vector | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "ovn_controller" "container" "vector" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - vector + - --config + - /etc/vector/vector.toml + volumeMounts: + - name: vector-config + mountPath: /etc/vector + - name: logs + mountPath: /logs + - name: vector-data + mountPath: /var/lib/vector + {{- end }} + {{- if .Values.pod.sidecars.ovn_logging_parser }} + - name: log-parser +{{ tuple $envAll "ovn_logging_parser" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.ovn_logging_parser | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "ovn_controller" "container" "ovn_logging_parser" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/ovn-network-logging-parser.sh + - start + env: + - name: VECTOR_HTTP_ENDPOINT + value: http://localhost:5001 + ports: + - name: http + containerPort: {{ tuple "ovn_logging_parser" "service" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + protocol: TCP + volumeMounts: + - name: neutron-etc + mountPath: /etc/neutron/neutron.conf + subPath: neutron.conf + readOnly: true + - name: ovn-bin + mountPath: /tmp/ovn-network-logging-parser.sh + subPath: ovn-network-logging-parser.sh + readOnly: true + - name: ovn-etc + mountPath: /etc/neutron/neutron-ovn-network-logging-parser-uwsgi.ini + subPath: neutron-ovn-network-logging-parser-uwsgi.ini + readOnly: true + {{- end }} + volumes: + - name: ovn-bin + configMap: + name: ovn-bin + defaultMode: 0777 + - name: run-openvswitch + hostPath: + path: /run/openvswitch + type: DirectoryOrCreate + - name: ovn-etc + secret: + secretName: {{ $configMapName }} + defaultMode: 0444 + - name: logs + hostPath: + path: /var/log/ovn + type: DirectoryOrCreate + - name: run-ovn + hostPath: + path: /run/ovn + type: DirectoryOrCreate + - name: gw-enabled + emptyDir: {} + {{- if .Values.pod.sidecars.vector }} + - name: vector-config + secret: + secretName: ovn-vector-config + - name: vector-data + emptyDir: {} + {{- end }} + {{- if .Values.pod.sidecars.ovn_logging_parser }} + - name: neutron-etc + secret: + secretName: neutron-etc + defaultMode: 0444 + {{- end }} +{{- end }} +{{- end }} + + +{{- if .Values.manifests.daemonset_ovn_controller }} +{{- $envAll := . }} +{{- $daemonset := "controller" }} +{{- $configMapName := "ovn-etc" }} +{{- $serviceAccountName := "ovn-controller" }} + +{{ tuple $envAll "ovn_controller" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +{{- $configmap_yaml := "ovn.configmap.etc" }} + +{{/* Preffer using .Values.overrides rather than .Values.conf.overrides */}} +{{- list $daemonset "ovn.daemonset" $serviceAccountName $configmap_yaml $configMapName "ovn.configmap.bin" "ovn-bin" . | include "helm-toolkit.utils.daemonset_overrides_root" }} + +{{- $serviceAccountNamespace := $envAll.Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: ovn-controller-list-nodes-role-{{ $serviceAccountNamespace }} +rules: +- apiGroups: [""] + resources: ["nodes"] + verbs: ["list", "get"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: ovn-controller-list-nodes-rolebinding-{{ $serviceAccountNamespace }} +subjects: +- kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $serviceAccountNamespace }} +roleRef: + kind: ClusterRole + name: ovn-controller-list-nodes-role-{{ $serviceAccountNamespace }} + apiGroup: rbac.authorization.k8s.io + +{{- end }} + diff --git a/ovn/templates/deployment-northd.yaml b/ovn/templates/deployment-northd.yaml new file mode 100644 index 0000000000..2dbbb68902 --- /dev/null +++ b/ovn/templates/deployment-northd.yaml @@ -0,0 +1,81 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "northdReadinessProbeTemplate" }} +exec: + command: + - /usr/bin/ovn-kube-util + - readiness-probe + - -t + - ovn-northd +{{- end }} + +{{- if .Values.manifests.deployment_northd }} +{{- $envAll := . }} + +{{- $serviceAccountName := "ovn-northd" }} +{{ tuple $envAll "ovn_northd" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: ovn-northd + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "ovn" "ovn-northd" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + replicas: {{ .Values.pod.replicas.ovn_northd }} + selector: + matchLabels: +{{ tuple $envAll "ovn" "ovn-northd" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} + template: + metadata: + labels: +{{ tuple $envAll "ovn" "ovn-northd" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + spec: + serviceAccountName: {{ $serviceAccountName }} + nodeSelector: + {{ .Values.labels.ovn_northd.node_selector_key }}: {{ .Values.labels.ovn_northd.node_selector_value }} + initContainers: +{{- tuple $envAll "ovn_northd" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: northd + command: + - /root/ovnkube.sh + - run-ovn-northd +{{ tuple $envAll "ovn_northd" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.ovn_northd | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "ovn_northd" "container" "northd" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} +{{ dict "envAll" . "component" "ovn_northd" "container" "northd" "type" "readiness" "probeTemplate" (include "northdReadinessProbeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }} +{{ dict "envAll" . "component" "ovn_northd" "container" "northd" "type" "liveness" "probeTemplate" (include "northdReadinessProbeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }} + env: + - name: OVN_DAEMONSET_VERSION + value: "3" + - name: OVN_LOGLEVEL_NORTHD + value: "-vconsole:info -vfile:info" + - name: OVN_KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OVN_KUBERNETES_NB_STATEFULSET + value: ovn-ovsdb-nb + - name: OVN_KUBERNETES_SB_STATEFULSET + value: ovn-ovsdb-sb + - name: OVN_SSL_ENABLE + value: "no" +{{- end }} diff --git a/ovn/templates/role-controller.yaml b/ovn/templates/role-controller.yaml new file mode 100644 index 0000000000..4ab9e8863f --- /dev/null +++ b/ovn/templates/role-controller.yaml @@ -0,0 +1,27 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: ovn-controller + namespace: {{ .Release.Namespace }} +rules: +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list diff --git a/ovn/templates/role-northd.yaml b/ovn/templates/role-northd.yaml new file mode 100644 index 0000000000..58d66e92cb --- /dev/null +++ b/ovn/templates/role-northd.yaml @@ -0,0 +1,27 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: ovn-northd + namespace: {{ .Release.Namespace }} +rules: +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list diff --git a/ovn/templates/role-ovsdb.yaml b/ovn/templates/role-ovsdb.yaml new file mode 100644 index 0000000000..f435ac8677 --- /dev/null +++ b/ovn/templates/role-ovsdb.yaml @@ -0,0 +1,35 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: ovn-ovsdb + namespace: {{ .Release.Namespace }} +rules: +- apiGroups: + - "apps" + resources: + - statefulsets + verbs: + - get +- apiGroups: + - "" + resources: + - pods + - endpoints + verbs: + - list + - get diff --git a/ovn/templates/rolebinding-controller.yaml b/ovn/templates/rolebinding-controller.yaml new file mode 100644 index 0000000000..64615eb08c --- /dev/null +++ b/ovn/templates/rolebinding-controller.yaml @@ -0,0 +1,31 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ovn-controller + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ovn-controller +subjects: +- kind: ServiceAccount + name: ovn-controller + namespace: {{ .Release.Namespace }} +- kind: ServiceAccount + name: ovn-controller-gw + namespace: {{ .Release.Namespace }} diff --git a/ovn/templates/rolebinding-northd.yaml b/ovn/templates/rolebinding-northd.yaml new file mode 100644 index 0000000000..537babe92e --- /dev/null +++ b/ovn/templates/rolebinding-northd.yaml @@ -0,0 +1,28 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ovn-northd + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ovn-northd +subjects: +- kind: ServiceAccount + name: ovn-northd + namespace: {{ .Release.Namespace }} diff --git a/ovn/templates/rolebinding-ovsdb.yaml b/ovn/templates/rolebinding-ovsdb.yaml new file mode 100644 index 0000000000..6211114a13 --- /dev/null +++ b/ovn/templates/rolebinding-ovsdb.yaml @@ -0,0 +1,31 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: ovn-ovsdb + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: ovn-ovsdb +subjects: +- kind: ServiceAccount + name: ovn-ovsdb-nb + namespace: {{ .Release.Namespace }} +- kind: ServiceAccount + name: ovn-ovsdb-sb + namespace: {{ .Release.Namespace }} diff --git a/ovn/templates/secret-vector.yaml b/ovn/templates/secret-vector.yaml new file mode 100644 index 0000000000..989f3afa3a --- /dev/null +++ b/ovn/templates/secret-vector.yaml @@ -0,0 +1,26 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.pod.sidecars.vector }} +{{- $envAll := . }} + +--- +apiVersion: v1 +kind: Secret +metadata: + name: ovn-vector-config +type: Opaque +data: +{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.vector "key" "vector.toml" "format" "Secret" ) | indent 2 }} +{{- end }} diff --git a/ovn/templates/service-ovsdb-nb.yaml b/ovn/templates/service-ovsdb-nb.yaml new file mode 100644 index 0000000000..56f7cd0969 --- /dev/null +++ b/ovn/templates/service-ovsdb-nb.yaml @@ -0,0 +1,31 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.service_ovn_ovsdb_nb }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "ovn-ovsdb-nb" "direct" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +spec: + publishNotReadyAddresses: true + ports: + - name: ovsdb + port: {{ tuple "ovn-ovsdb-nb" "internal" "ovsdb" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - name: raft + port: {{ tuple "ovn-ovsdb-nb" "internal" "raft" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + selector: +{{ tuple $envAll "ovn" "ovn-ovsdb-nb" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- end }} diff --git a/ovn/templates/service-ovsdb-sb.yaml b/ovn/templates/service-ovsdb-sb.yaml new file mode 100644 index 0000000000..4a6b5864df --- /dev/null +++ b/ovn/templates/service-ovsdb-sb.yaml @@ -0,0 +1,31 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.service_ovn_ovsdb_sb }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "ovn-ovsdb-sb" "direct" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +spec: + publishNotReadyAddresses: true + ports: + - name: ovsdb + port: {{ tuple "ovn-ovsdb-sb" "internal" "ovsdb" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - name: raft + port: {{ tuple "ovn-ovsdb-sb" "internal" "raft" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + selector: +{{ tuple $envAll "ovn" "ovn-ovsdb-sb" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- end }} diff --git a/ovn/templates/statefulset-ovsdb-nb.yaml b/ovn/templates/statefulset-ovsdb-nb.yaml new file mode 100644 index 0000000000..d19d5105d1 --- /dev/null +++ b/ovn/templates/statefulset-ovsdb-nb.yaml @@ -0,0 +1,131 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "ovnnbReadinessProbeTemplate" }} +exec: + command: + - /usr/bin/ovn-kube-util + - readiness-probe + - -t +{{- if gt (int .Values.pod.replicas.ovn_ovsdb_nb) 1 }} + - ovnnb-db-raft +{{- else }} + - ovnnb-db +{{- end }} +{{- end }} + +{{- if .Values.manifests.statefulset_ovn_ovsdb_nb }} +{{- $envAll := . }} + +{{- $serviceAccountName := "ovn-ovsdb-nb" }} +{{ tuple $envAll "ovn_ovsdb_nb" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: ovn-ovsdb-nb + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "ovn" "ovn-ovsdb-nb" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + serviceName: {{ tuple "ovn-ovsdb-nb" "direct" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + podManagementPolicy: Parallel + replicas: {{ .Values.pod.replicas.ovn_ovsdb_nb }} + selector: + matchLabels: +{{ tuple $envAll "ovn" "ovn-ovsdb-nb" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} + template: + metadata: + labels: +{{ tuple $envAll "ovn" "ovn-ovsdb-nb" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + spec: + serviceAccountName: {{ $serviceAccountName }} + affinity: +{{- tuple $envAll "ovn" "ovn-ovsdb-nb" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} + nodeSelector: + {{ .Values.labels.ovn_ovsdb_nb.node_selector_key }}: {{ .Values.labels.ovn_ovsdb_nb.node_selector_value }} + initContainers: +{{- tuple $envAll "ovn_ovsdb_nb" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: ovsdb + command: + - /root/ovnkube.sh +{{- if gt (int .Values.pod.replicas.ovn_ovsdb_nb) 1 }} + - nb-ovsdb-raft +{{- else }} + - nb-ovsdb +{{- end }} +{{ tuple $envAll "ovn_ovsdb_nb" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.ovn_ovsdb_nb | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" . "component" "ovn_ovsdb_nb" "container" "ovsdb" "type" "readiness" "probeTemplate" (include "ovnnbReadinessProbeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }} + ports: + - containerPort: {{ tuple "ovn-ovsdb-nb" "internal" "ovsdb" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - containerPort: {{ tuple "ovn-ovsdb-nb" "internal" "raft" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + env: + - name: OVN_DAEMONSET_VERSION + value: "3" + - name: OVN_LOGLEVEL_NB + value: "-vconsole:info -vfile:info" + - name: OVN_KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OVN_KUBERNETES_STATEFULSET + value: ovn-ovsdb-nb + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: OVN_SSL_ENABLE + value: "no" + - name: ENABLE_IPSEC + value: "false" + - name: OVN_NB_RAFT_ELECTION_TIMER + value: "1000" + - name: OVN_NB_PORT + value: {{ tuple "ovn-ovsdb-nb" "internal" "ovsdb" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: OVN_NB_RAFT_PORT + value: {{ tuple "ovn-ovsdb-nb" "internal" "raft" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + volumeMounts: + - name: run-openvswitch + mountPath: /var/run/openvswitch + - name: run-openvswitch + mountPath: /var/run/ovn + - name: data + mountPath: /etc/ovn + volumes: + - name: run-openvswitch + hostPath: + path: /run/openvswitch + type: DirectoryOrCreate +{{- if not .Values.volume.ovn_ovsdb_nb.enabled }} + - name: data + emptyDir: {} +{{- else }} + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: {{ $envAll.Values.volume.ovn_ovsdb_nb.class_name }} + resources: + requests: + storage: {{ $envAll.Values.volume.ovn_ovsdb_nb.size }} +{{- end }} + +{{- end }} diff --git a/ovn/templates/statefulset-ovsdb-sb.yaml b/ovn/templates/statefulset-ovsdb-sb.yaml new file mode 100644 index 0000000000..a6180aaac1 --- /dev/null +++ b/ovn/templates/statefulset-ovsdb-sb.yaml @@ -0,0 +1,131 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "ovnsbReadinessProbeTemplate" }} +exec: + command: + - /usr/bin/ovn-kube-util + - readiness-probe + - -t +{{- if gt (int .Values.pod.replicas.ovn_ovsdb_sb) 1 }} + - ovnsb-db-raft +{{- else }} + - ovnsb-db +{{- end }} +{{- end }} + +{{- if .Values.manifests.statefulset_ovn_ovsdb_sb }} +{{- $envAll := . }} + +{{- $serviceAccountName := "ovn-ovsdb-sb" }} +{{ tuple $envAll "ovn_ovsdb_sb" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: ovn-ovsdb-sb + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "ovn" "ovn-ovsdb-sb" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + serviceName: {{ tuple "ovn-ovsdb-sb" "direct" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + podManagementPolicy: Parallel + replicas: {{ .Values.pod.replicas.ovn_ovsdb_sb }} + selector: + matchLabels: +{{ tuple $envAll "ovn" "ovn-ovsdb-sb" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} + template: + metadata: + labels: +{{ tuple $envAll "ovn" "ovn-ovsdb-sb" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + spec: + serviceAccountName: {{ $serviceAccountName }} + affinity: +{{- tuple $envAll "ovn" "ovn-ovsdb-sb" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} + nodeSelector: + {{ .Values.labels.ovn_ovsdb_sb.node_selector_key }}: {{ .Values.labels.ovn_ovsdb_sb.node_selector_value }} + initContainers: +{{- tuple $envAll "ovn_ovsdb_sb" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: ovsdb + command: + - /root/ovnkube.sh +{{- if gt (int .Values.pod.replicas.ovn_ovsdb_sb) 1 }} + - sb-ovsdb-raft +{{- else }} + - sb-ovsdb +{{- end }} +{{ tuple $envAll "ovn_ovsdb_sb" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.ovn_ovsdb_sb | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" . "component" "ovn_ovsdb_sb" "container" "ovsdb" "type" "readiness" "probeTemplate" (include "ovnsbReadinessProbeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }} + ports: + - containerPort: {{ tuple "ovn-ovsdb-sb" "internal" "ovsdb" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - containerPort: {{ tuple "ovn-ovsdb-sb" "internal" "raft" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + env: + - name: OVN_DAEMONSET_VERSION + value: "3" + - name: OVN_LOGLEVEL_SB + value: "-vconsole:info -vfile:info" + - name: OVN_KUBERNETES_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: OVN_KUBERNETES_STATEFULSET + value: ovn-ovsdb-sb + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: OVN_SSL_ENABLE + value: "no" + - name: ENABLE_IPSEC + value: "false" + - name: OVN_SB_RAFT_ELECTION_TIMER + value: "1000" + - name: OVN_SB_PORT + value: {{ tuple "ovn-ovsdb-sb" "internal" "ovsdb" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: OVN_SB_RAFT_PORT + value: {{ tuple "ovn-ovsdb-sb" "internal" "raft" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + volumeMounts: + - name: run-openvswitch + mountPath: /var/run/openvswitch + - name: run-openvswitch + mountPath: /var/run/ovn + - name: data + mountPath: /etc/ovn + volumes: + - name: run-openvswitch + hostPath: + path: /run/openvswitch + type: DirectoryOrCreate +{{- if not .Values.volume.ovn_ovsdb_sb.enabled }} + - name: data + emptyDir: {} +{{- else }} + volumeClaimTemplates: + - metadata: + name: data + spec: + accessModes: ["ReadWriteOnce"] + storageClassName: {{ $envAll.Values.volume.ovn_ovsdb_sb.class_name }} + resources: + requests: + storage: {{ $envAll.Values.volume.ovn_ovsdb_sb.size }} +{{- end }} + +{{- end }} diff --git a/ovn/values.yaml b/ovn/values.yaml new file mode 100644 index 0000000000..49d4af8961 --- /dev/null +++ b/ovn/values.yaml @@ -0,0 +1,444 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for openvswitch. +# This is a YAML-formatted file. +# Declare name/value pairs to be passed into your templates. +# name: value + +--- +release_group: null + +images: + tags: + ovn_ovsdb_nb: docker.io/openstackhelm/ovn:ubuntu_focal + ovn_ovsdb_sb: docker.io/openstackhelm/ovn:ubuntu_focal + ovn_northd: docker.io/openstackhelm/ovn:ubuntu_focal + ovn_controller: docker.io/openstackhelm/ovn:ubuntu_focal + ovn_controller_kubectl: docker.io/openstackhelm/ceph-config-helper:latest-ubuntu_jammy + dep_check: quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal + image_repo_sync: docker.io/library/docker:17.07.0 + vector: docker.io/timberio/vector:0.39.0-debian + ovn_logging_parser: docker.io/openstackhelm/neutron:2024.1-ubuntu_jammy + pull_policy: "IfNotPresent" + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +labels: + ovn_ovsdb_nb: + node_selector_key: openstack-network-node + node_selector_value: enabled + ovn_ovsdb_sb: + node_selector_key: openstack-network-node + node_selector_value: enabled + ovn_northd: + node_selector_key: openstack-network-node + node_selector_value: enabled + ovn_controller: + node_selector_key: openvswitch + node_selector_value: enabled + ovn_controller_gw: + node_selector_key: l3-agent + node_selector_value: enabled + +volume: + ovn_ovsdb_nb: + enabled: true + class_name: general + size: 5Gi + ovn_ovsdb_sb: + enabled: true + class_name: general + size: 5Gi + +network: + interface: + # Tunnel interface will be used for VXLAN tunneling. + tunnel: null + # If tunnel is null there is a fallback mechanism to search + # for interface with routing using tunnel network cidr. + tunnel_network_cidr: "0/0" + +conf: + ovn_cms_options: "availability-zones=nova" + ovn_cms_options_gw_enabled: "enable-chassis-as-gw,availability-zones=nova" + ovn_encap_type: geneve + ovn_bridge: br-int + ovn_bridge_mappings: external:br-ex + # For DPDK enabled environments, enable netdev datapath type for br-int + # ovn_bridge_datapath_type: netdev + + # auto_bridge_add: + # br-private: eth0 + # br-public: eth1 + auto_bridge_add: {} + ovs_user_name: openvswitch + ovn_network_logging_parser_uwsgi: + uwsgi: + add-header: "Connection: close" + buffer-size: 65535 + die-on-term: true + enable-threads: true + exit-on-reload: false + hook-master-start: unix_signal:15 gracefully_kill_them_all + lazy-apps: true + log-x-forwarded-for: true + master: true + processes: 1 + procname-prefix-spaced: "neutron-ovn-network-logging-parser:" + route-user-agent: '^kube-probe.* donotlog:' + thunder-lock: true + worker-reload-mercy: 80 + wsgi-file: /var/lib/openstack/bin/neutron-ovn-network-logging-parser-wsgi + vector: | + [sources.file_logs] + type = "file" + include = [ "/logs/ovn-controller.log" ] + + [sinks.ovn_log_parser_in] + type = "http" + inputs = ["file_logs"] + uri = "{{ tuple "ovn_logging_parser" "default" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" }}" + encoding.codec = "json" + method = "post" + + [sources.ovn_log_parser_out] + type = "http_server" + address = "0.0.0.0:5001" + encoding = "json" + + [transforms.parse_log_message] + type = "remap" + inputs = ["ovn_log_parser_out"] + source = ''' + del(.source_type) + del(.path) + ''' + + [sinks.loki_sink] + type = "loki" + labels.event_source = "network_logs" + inputs = ["parse_log_message"] + endpoint = "http://loki.monitoring:3100" + encoding.codec = "json" + tenant_id = "{{`{{ project_id }}`}}" + +pod: + # NOTE: should be same as nova.pod.use_fqdn.compute + use_fqdn: + compute: true + security_context: + ovn_northd: + container: + northd: + capabilities: + add: + - SYS_NICE + ovn_controller: + container: + controller_init: + readOnlyRootFilesystem: true + privileged: true + controller: + readOnlyRootFilesystem: true + privileged: true + ovn_logging_parser: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + vector: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + tolerations: + ovn_ovsdb_nb: + enabled: false + ovn_ovsdb_sb: + enabled: false + ovn_northd: + enabled: false + ovn_controller: + enabled: false + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + weight: + default: 10 + + probes: + ovn_northd: + northd: + readiness: + enabled: true + params: + initialDelaySeconds: 30 + timeoutSeconds: 30 + periodSeconds: 60 + ovn_ovsdb_nb: + ovsdb: + readiness: + enabled: true + params: + initialDelaySeconds: 30 + timeoutSeconds: 30 + periodSeconds: 60 + ovn_ovsdb_sb: + ovsdb: + readiness: + enabled: true + params: + initialDelaySeconds: 30 + timeoutSeconds: 30 + periodSeconds: 60 + ovn_controller: + controller: + readiness: + enabled: true + params: + initialDelaySeconds: 30 + timeoutSeconds: 30 + periodSeconds: 60 + ovn_controller_gw: + controller: + readiness: + enabled: true + params: + initialDelaySeconds: 30 + timeoutSeconds: 30 + periodSeconds: 60 + dns_policy: "ClusterFirstWithHostNet" + replicas: + ovn_ovsdb_nb: 1 + ovn_ovsdb_sb: 1 + ovn_northd: 1 + lifecycle: + upgrades: + daemonsets: + pod_replacement_strategy: RollingUpdate + ovn_ovsdb_nb: + enabled: true + min_ready_seconds: 0 + max_unavailable: 1 + ovn_ovsdb_sb: + enabled: true + min_ready_seconds: 0 + max_unavailable: 1 + ovn_northd: + enabled: true + min_ready_seconds: 0 + max_unavailable: 1 + ovn_controller: + enabled: true + min_ready_seconds: 0 + max_unavailable: 1 + resources: + enabled: false + ovn_ovsdb_nb: + requests: + memory: "384Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "1000m" + ovn_ovsdb_sb: + requests: + memory: "384Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "1000m" + ovn_northd: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + ovn_controller: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + ovn_logging_parser: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "256Mi" + cpu: "500m" + vector: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "256Mi" + cpu: "500m" + jobs: + image_repo_sync: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + + sidecars: + ovn_logging_parser: false + vector: false + +secrets: + oci_image_registry: + ovn: ovn-oci-image-registry-key + +# TODO: Check these endpoints?! +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + openvswitch: + username: openvswitch + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + ovn_ovsdb_nb: + name: ovn-ovsdb-nb + namespace: null + hosts: + default: ovn-ovsdb-nb + host_fqdn_override: + default: null + port: + ovsdb: + default: 6641 + raft: + default: 6643 + ovn_ovsdb_sb: + name: ovn-ovsdb-sb + namespace: null + hosts: + default: ovn-ovsdb-sb + host_fqdn_override: + default: null + port: + ovsdb: + default: 6642 + raft: + default: 6644 + ovn_logging_parser: + name: ovn-logging-parser + namespace: null + hosts: + default: localhost + host_fqdn_override: + default: localhost + scheme: + default: 'http' + service: 'http' + path: + default: "/logs" + port: + api: + default: 9697 + service: 9697 + +network_policy: + ovn_ovsdb_nb: + ingress: + - {} + egress: + - {} + ovn_ovsdb_sb: + ingress: + - {} + egress: + - {} + ovn_northd: + ingress: + - {} + egress: + - {} + ovn_controller: + ingress: + - {} + egress: + - {} + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - openvswitch-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + ovn_ovsdb_nb: null + ovn_ovsdb_sb: null + ovn_northd: + services: + - endpoint: internal + service: ovn-ovsdb-nb + - endpoint: internal + service: ovn-ovsdb-sb + ovn_controller: + services: + - endpoint: internal + service: ovn-ovsdb-sb + pod: + - requireSameNode: true + labels: + application: openvswitch + component: server + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + +manifests: + configmap_bin: true + configmap_etc: true + deployment_northd: true + service_ovn_ovsdb_nb: true + service_ovn_ovsdb_sb: true + statefulset_ovn_ovsdb_nb: true + statefulset_ovn_ovsdb_sb: true + deployment_ovn_northd: true + daemonset_ovn_controller: true + job_image_repo_sync: true +... diff --git a/playbooks/build-chart.yaml b/playbooks/build-chart.yaml new file mode 100644 index 0000000000..02fd205d56 --- /dev/null +++ b/playbooks/build-chart.yaml @@ -0,0 +1,32 @@ +--- +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- hosts: all + roles: + - ensure-python + - ensure-pip + - name: ensure-helm + helm_version: "3.16.4" + + tasks: + - name: Install reno + pip: + name: reno>=4.1.0 + extra_args: "--ignore-installed" + become: yes + + - name: make all + make: + chdir: "{{ zuul.project.src_dir }}" + target: all +... diff --git a/playbooks/deploy-env.yaml b/playbooks/deploy-env.yaml new file mode 100644 index 0000000000..dd26203b27 --- /dev/null +++ b/playbooks/deploy-env.yaml @@ -0,0 +1,25 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- hosts: all + strategy: linear + become: true + gather_facts: true + roles: + - ensure-python + - ensure-pip + - clear-firewall + - deploy-apparmor + - deploy-selenium + - deploy-env +... diff --git a/playbooks/enable-hugepages.yaml b/playbooks/enable-hugepages.yaml new file mode 100644 index 0000000000..186c07671f --- /dev/null +++ b/playbooks/enable-hugepages.yaml @@ -0,0 +1,20 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- hosts: all + gather_facts: True + become: yes + roles: + - role: enable-hugepages + when: hugepages.enabled|default(false)|bool == true +... diff --git a/playbooks/inject-keys.yaml b/playbooks/inject-keys.yaml new file mode 100644 index 0000000000..c9a85b2612 --- /dev/null +++ b/playbooks/inject-keys.yaml @@ -0,0 +1,11 @@ +--- +- hosts: all + tasks: + - name: Put keys to .ssh/authorized_keys + lineinfile: + path: /home/zuul/.ssh/authorized_keys + state: present + line: "{{ item }}" + loop: + - "ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBMyM6sgu/Xgg+VaLJX5c6gy6ynYX7pO7XNobnKotYRulcEkmiLprvLSg+WP25VDAcSoif3rek3qiVnEYh6R2/Go= vlad@russell" +... diff --git a/playbooks/lint.yml b/playbooks/lint.yml new file mode 100644 index 0000000000..db41259587 --- /dev/null +++ b/playbooks/lint.yml @@ -0,0 +1,89 @@ +--- +# Copyright 2018 SUSE LINUX GmbH. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- hosts: all + roles: + - ensure-python + - ensure-pip + - name: ensure-helm + helm_version: "3.16.4" + - name: ensure-chart-testing + chart_testing_version: "3.11.0" + - name: chart-testing + chart_testing_options: "--target-branch=master --chart-dirs=. --validate-maintainers=false --check-version-increment=false" + zuul_work_dir: "{{ work_dir }}" + vars: + work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}" + + tasks: + - name: Install reno + pip: + name: reno>=4.1.0 + extra_args: "--ignore-installed" + become: yes + + - name: make all + make: + chdir: "{{ zuul.project.src_dir }}" + target: all + + - name: make all osh + make: + chdir: "{{ zuul.project.src_dir }}/{{ zuul_osh_relative_path | default('../openstack-helm/') }}" + target: all + when: lint_osh is defined + + - name: Prevent trailing whitespaces + shell: find . \! \( -path "*/\.*" -o -path "*/doc/build/*" -o -name "*.tgz" -o -name "*.png" -o -name "*.jpg" \) -type f -exec grep -El " +$" {} \; + register: _found_whitespaces + failed_when: _found_whitespaces.stdout != "" + args: + chdir: "{{ ansible_user_dir }}/src/{{ zuul.project.canonical_name }}" + + - name: Check release note version matches + shell: ../openstack-helm-infra/tools/gate/reno-check.sh + args: + chdir: "{{ ansible_user_dir }}/src/{{ zuul.project.canonical_name }}" + + - name: Check if yamllint.conf exists + stat: + path: "{{ ansible_user_dir }}/src/{{ zuul.project.canonical_name }}/yamllint.conf" + register: yamllintconf + + - name: Install jq and pip + apt: + pkg: + - jq + - python3-pip + become: yes + when: yamllintconf.stat.exists == True + + - name: Install tox + shell: pip3 install -U tox + become: yes + when: yamllintconf.stat.exists == True + + - name: Execute yamllint check for values* yaml files + command: tox -e lint + args: + chdir: "{{ ansible_user_dir }}/src/{{ zuul.project.canonical_name }}" + when: yamllintconf.stat.exists == True + + - name: Execute yamllint check for osh values* yaml files + command: tox -e lint + args: + chdir: "{{ zuul.project.src_dir }}/{{ zuul_osh_relative_path | default('../openstack-helm/') }}" + when: yamllintconf.stat.exists == True and lint_osh is defined +... diff --git a/playbooks/mount-volumes.yaml b/playbooks/mount-volumes.yaml new file mode 100644 index 0000000000..0049da194e --- /dev/null +++ b/playbooks/mount-volumes.yaml @@ -0,0 +1,17 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- hosts: all + roles: + - mount-extra-volume +... diff --git a/playbooks/osh-infra-bandit.yaml b/playbooks/osh-infra-bandit.yaml new file mode 100644 index 0000000000..b77fa586b8 --- /dev/null +++ b/playbooks/osh-infra-bandit.yaml @@ -0,0 +1,19 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- hosts: primary + roles: + - ensure-python + - ensure-pip + - osh-bandit +... diff --git a/playbooks/osh-infra-collect-logs.yaml b/playbooks/osh-infra-collect-logs.yaml new file mode 100644 index 0000000000..83e768877e --- /dev/null +++ b/playbooks/osh-infra-collect-logs.yaml @@ -0,0 +1,43 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- hosts: all + vars_files: + - vars.yaml + vars: + work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}" + logs_dir: "/tmp/logs" + roles: + - gather-host-logs + tags: + - gather-host-logs + +- hosts: primary + vars_files: + - vars.yaml + vars: + work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}" + logs_dir: "/tmp/logs" + roles: + - helm-release-status + - describe-kubernetes-objects + - gather-pod-logs + - gather-prom-metrics + - gather-selenium-data + tags: + - helm-release-status + - describe-kubernetes-objects + - gather-pod-logs + - gather-prom-metrics + - gather-selenium-data +... diff --git a/playbooks/osh-infra-deploy-docker.yaml b/playbooks/osh-infra-deploy-docker.yaml new file mode 100644 index 0000000000..785617dbe6 --- /dev/null +++ b/playbooks/osh-infra-deploy-docker.yaml @@ -0,0 +1,43 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- hosts: all + vars_files: + - vars.yaml + vars: + work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}" + gather_facts: False + become: yes + roles: + - deploy-python + tags: + - deploy-python + +- hosts: all + vars_files: + - vars.yaml + vars: + work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}" + gather_facts: True + become: yes + roles: + - setup-firewall + - deploy-python-pip + - deploy-docker + - deploy-jq + tags: + - setup-firewall + - deploy-python-pip + - deploy-docker + - deploy-jq +... diff --git a/playbooks/osh-infra-deploy-selenium.yaml b/playbooks/osh-infra-deploy-selenium.yaml new file mode 100644 index 0000000000..650507b95e --- /dev/null +++ b/playbooks/osh-infra-deploy-selenium.yaml @@ -0,0 +1,26 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- hosts: primary + vars_files: + - vars.yaml + vars: + work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}" + gather_facts: True + become: yes + roles: + - ensure-pip + - deploy-selenium + tags: + - deploy-selenium +... diff --git a/playbooks/osh-infra-gate-runner.yaml b/playbooks/osh-infra-gate-runner.yaml new file mode 100644 index 0000000000..cecd684a4c --- /dev/null +++ b/playbooks/osh-infra-gate-runner.yaml @@ -0,0 +1,47 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- hosts: primary + tasks: + - name: Ensure pip + include_role: + name: ensure-pip + - name: Clear firewall + include_role: + name: clear-firewall + - name: Override images + include_role: + name: override-images + when: buildset_registry is defined + - name: Use docker mirror + include_role: + name: use-docker-mirror + - name: "creating directory for run artifacts" + file: + path: "/tmp/artifacts" + state: directory + + - name: Run gate scripts + include_role: + name: "{{ ([item] | flatten | length == 1) | ternary('osh-run-script', 'osh-run-script-set') }}" + vars: + workload: "{{ [item] | flatten }}" + loop: "{{ gate_scripts }}" + + - name: "Downloads artifacts to executor" + synchronize: + src: "/tmp/artifacts" + dest: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}" + mode: pull + ignore_errors: True +... diff --git a/playbooks/osh-infra-upgrade-host.yaml b/playbooks/osh-infra-upgrade-host.yaml new file mode 100644 index 0000000000..0807eae5e3 --- /dev/null +++ b/playbooks/osh-infra-upgrade-host.yaml @@ -0,0 +1,53 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- hosts: all + vars_files: + - vars.yaml + vars: + work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}" + gather_facts: False + become: yes + roles: + - deploy-python + tags: + - deploy-python + +- hosts: all + vars_files: + - vars.yaml + vars: + work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}" + gather_facts: True + become: yes + roles: + - upgrade-host + - start-zuul-console + - disable-local-nameserver + tags: + - upgrade-host + - start-zuul-console + - disable-local-nameserver + +- hosts: all + vars_files: + - vars.yaml + vars: + work_dir: "{{ zuul.project.src_dir }}/{{ zuul_osh_infra_relative_path | default('') }}" + gather_facts: False + become: yes + roles: + - deploy-apparmor + tags: + - deploy-apparmor +... diff --git a/playbooks/prepare-hosts.yaml b/playbooks/prepare-hosts.yaml new file mode 100644 index 0000000000..c64aa0d655 --- /dev/null +++ b/playbooks/prepare-hosts.yaml @@ -0,0 +1,17 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- hosts: all + roles: + - start-zuul-console +... diff --git a/playbooks/publish/post.yaml b/playbooks/publish/post.yaml new file mode 100644 index 0000000000..61f3a45065 --- /dev/null +++ b/playbooks/publish/post.yaml @@ -0,0 +1,55 @@ +--- +# Copyright 2020 VEXXHOST, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- hosts: all + tasks: + - name: Download current index + register: _get_url + failed_when: _get_url.status_code not in (200, 404) + get_url: + url: "https://tarballs.opendev.org/{{ zuul.project.name }}/index.yaml" + dest: "{{ zuul.project.src_dir }}/index.yaml" + + - name: Create a new index + when: _get_url.status_code == 404 + shell: helm repo index {{ zuul.project.src_dir }} --url https://tarballs.opendev.org/{{ zuul.project.name }} + + - name: Merge into existing index + when: _get_url.status_code == 200 + shell: helm repo index {{ zuul.project.src_dir }} --merge {{ zuul.project.src_dir }}/index.yaml --url https://tarballs.opendev.org/{{ zuul.project.name }} + + - name: Ensure artifact directory exists + file: + path: "{{ zuul.executor.work_root }}/artifacts/" + state: directory + delegate_to: localhost + + - name: Gather the artifacts + find: + file_type: file + paths: "{{ zuul.project.src_dir }}" + patterns: "*.tar.gz,*.tgz,index.yaml" + register: result + + - name: Update Helm repository + synchronize: + mode: pull + src: "{{ item.path }}" + dest: "{{ zuul.executor.work_root }}/artifacts/" + verify_host: true + owner: no + group: no + with_items: "{{ result.files }}" +... diff --git a/playbooks/publish/run.yaml b/playbooks/publish/run.yaml new file mode 100644 index 0000000000..50d0695cf4 --- /dev/null +++ b/playbooks/publish/run.yaml @@ -0,0 +1,20 @@ +--- +# Copyright 2020 VEXXHOST, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- hosts: all + roles: + - name: build-helm-packages + work_dir: "{{ zuul.project.src_dir }}" +... diff --git a/playbooks/roles b/playbooks/roles new file mode 120000 index 0000000000..d8c4472ca1 --- /dev/null +++ b/playbooks/roles @@ -0,0 +1 @@ +../roles \ No newline at end of file diff --git a/playbooks/run-scripts.yaml b/playbooks/run-scripts.yaml new file mode 100644 index 0000000000..180ca0bdec --- /dev/null +++ b/playbooks/run-scripts.yaml @@ -0,0 +1,76 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- hosts: primary + tasks: + - name: Override images + when: buildset_registry is defined + vars: + work_dir: "{{ zuul.project.src_dir }}" + block: + - name: Buildset registry alias + include_role: + name: deploy-env + tasks_from: buildset_registry_alias + + - name: Print zuul + debug: + var: zuul + + - name: Override proposed images from artifacts + shell: > + find {{ override_paths | join(" ") }} -type f -exec sed -Ei + "s#['\"]?docker\.io/({{ repo }}):({{ tag }})['\"]?\$#{{ buildset_registry_alias }}:{{ buildset_registry.port }}/\1:\2#g" {} + + loop: "{{ zuul.artifacts | default([]) }}" + args: + chdir: "{{ work_dir }}" + loop_control: + loop_var: zj_zuul_artifact + when: "'metadata' in zj_zuul_artifact and zj_zuul_artifact.metadata.type | default('') == 'container_image'" + vars: + tag: "{{ zj_zuul_artifact.metadata.tag }}" + repo: "{{ zj_zuul_artifact.metadata.repository }}" + override_paths: + - ../openstack-helm*/values_overrides + - ../openstack-helm*/*/values* + - ../openstack-helm-infra/tools/deployment/ + + - name: Diff + shell: | + set -ex; + for dir in openstack-helm openstack-helm-infra; do + path="{{ work_dir }}/../${dir}/" + if [ ! -d "${path}" ]; then continue; fi + echo "${dir} diff" + cd "${path}"; git diff; cd -; + done + + - name: "creating directory for run artifacts" + file: + path: "/tmp/artifacts" + state: directory + + - name: Run gate scripts + include_role: + name: "{{ ([item] | flatten | length == 1) | ternary('osh-run-script', 'osh-run-script-set') }}" + vars: + workload: "{{ [item] | flatten }}" + loop: "{{ gate_scripts }}" + + - name: "Downloads artifacts to executor" + synchronize: + src: "/tmp/artifacts" + dest: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}" + mode: pull + ignore_errors: True +... diff --git a/playbooks/vars.yaml b/playbooks/vars.yaml new file mode 100644 index 0000000000..8f6d99e7fa --- /dev/null +++ b/playbooks/vars.yaml @@ -0,0 +1,17 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# NOTE(portdirect): for use in the dev-deploy scripts, a valid vars.yaml is +# required, so provide some nonsense, yet harmless input. +--- +ansible_python_interpreter: python3 +... diff --git a/postgresql/.helmignore b/postgresql/.helmignore new file mode 100644 index 0000000000..f0c1319444 --- /dev/null +++ b/postgresql/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/postgresql/Chart.yaml b/postgresql/Chart.yaml new file mode 100644 index 0000000000..ffe66ec0be --- /dev/null +++ b/postgresql/Chart.yaml @@ -0,0 +1,29 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: v14.5 +description: OpenStack-Helm PostgreSQL +name: postgresql +version: 2024.2.0 +home: https://www.postgresql.org +sources: + - https://github.com/postgres/postgres + - https://opendev.org/openstack/openstack-helm +maintainers: + - name: OpenStack-Helm Authors +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit + version: ">= 0.1.0" +... diff --git a/postgresql/templates/bin/_backup_postgresql.sh.tpl b/postgresql/templates/bin/_backup_postgresql.sh.tpl new file mode 100755 index 0000000000..a9ea35c35a --- /dev/null +++ b/postgresql/templates/bin/_backup_postgresql.sh.tpl @@ -0,0 +1,94 @@ +#!/bin/bash + +SCOPE=${1:-"all"} + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# This is needed to get the postgresql admin password +# Turn off tracing so the password doesn't get printed. +set +x +export PGPASSWORD=$(cat /etc/postgresql/admin_user.conf \ + | grep postgres | awk -F: '{print $5}') + +# Note: not using set -e in this script because more elaborate error handling +# is needed. + +source /tmp/backup_main.sh + +# Export the variables required by the framework +# Note: REMOTE_BACKUP_ENABLED and CONTAINER_NAME are already exported +export DB_NAMESPACE=${POSTGRESQL_POD_NAMESPACE} +export DB_NAME="postgres" +export LOCAL_DAYS_TO_KEEP=$POSTGRESQL_LOCAL_BACKUP_DAYS_TO_KEEP +export REMOTE_DAYS_TO_KEEP=$POSTGRESQL_REMOTE_BACKUP_DAYS_TO_KEEP +export REMOTE_BACKUP_RETRIES=${NUMBER_OF_RETRIES_SEND_BACKUP_TO_REMOTE} +export MIN_DELAY_SEND_REMOTE=${MIN_DELAY_SEND_BACKUP_TO_REMOTE} +export MAX_DELAY_SEND_REMOTE=${MAX_DELAY_SEND_BACKUP_TO_REMOTE} +export ARCHIVE_DIR=${POSTGRESQL_BACKUP_BASE_DIR}/db/${DB_NAMESPACE}/${DB_NAME}/archive + +# This function dumps all database files to the $TMP_DIR that is being +# used as a staging area for preparing the backup tarball. Log file to +# write to is passed in - the framework will expect that file to have any +# errors that occur if the database dump is unsuccessful, so that it can +# add the file contents to its own logs. +dump_databases_to_directory() { + TMP_DIR=$1 + LOG_FILE=$2 + SCOPE=${3:-"all"} + + PG_DUMP_OPTIONS=$(echo $POSTGRESQL_BACKUP_PG_DUMPALL_OPTIONS | sed 's/"//g') + PG_DUMP="pg_dump \ + $PG_DUMP_OPTIONS --create \ + -U $POSTGRESQL_ADMIN_USER \ + -h $POSTGRESQL_SERVICE_HOST" + PG_DUMPALL="pg_dumpall \ + $PG_DUMP_OPTIONS \ + -U $POSTGRESQL_ADMIN_USER \ + -h $POSTGRESQL_SERVICE_HOST" + + SQL_FILE=postgres.${POSTGRESQL_POD_NAMESPACE}.${SCOPE} + + cd $TMP_DIR + + if [[ "${SCOPE}" == "all" ]]; then + # Dump all databases + ${PG_DUMPALL} --file=${TMP_DIR}/${SQL_FILE}.sql 2>>${LOG_FILE} + else + if [[ "${SCOPE}" != "postgres" && "${SCOPE}" != "template0" && "${SCOPE}" != "template1" ]]; then + # Dump the specified database + ${PG_DUMP} --file=${TMP_DIR}/${SQL_FILE}.sql ${SCOPE} 2>>${LOG_FILE} + else + log ERROR "It is not allowed to backup up the ${SCOPE} database." + return 1 + fi + fi + + if [[ $? -eq 0 && -s "${TMP_DIR}/${SQL_FILE}.sql" ]]; then + log INFO postgresql_backup "Database(s) dumped successfully. (SCOPE = ${SCOPE})" + return 0 + else + log ERROR "Backup of the postgresql database(s) failed and needs attention. (SCOPE = ${SCOPE})" + return 1 + fi +} + +# Verify all the databases backup archives +verify_databases_backup_archives() { + #################################### + # TODO: add implementation of local backup verification + #################################### + return 0 +} + +# Call main program to start the database backup +backup_databases ${SCOPE} diff --git a/postgresql/templates/bin/_common_backup_restore.sh.tpl b/postgresql/templates/bin/_common_backup_restore.sh.tpl new file mode 100644 index 0000000000..39e725ba8e --- /dev/null +++ b/postgresql/templates/bin/_common_backup_restore.sh.tpl @@ -0,0 +1,94 @@ +#!/bin/bash + +# Copyright 2018 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Do not use set -x here because the manual backup or restore pods may be using +# these functions, and it will distort the command output to have tracing on. + +log_backup_error_exit() { + MSG=$1 + ERRCODE=$2 + log ERROR postgresql_backup "${MSG}" + exit $ERRCODE +} + +log() { + #Log message to a file or stdout + #TODO: This can be convert into mail alert of alert send to a monitoring system + #Params: $1 log level + #Params: $2 service + #Params: $3 message + #Params: $4 Destination + LEVEL=$1 + SERVICE=$2 + MSG=$3 + DEST=$4 + DATE=$(date +"%m-%d-%y %H:%M:%S") + if [ -z "$DEST" ] + then + echo "${DATE} ${LEVEL}: $(hostname) ${SERVICE}: ${MSG}" + else + echo "${DATE} ${LEVEL}: $(hostname) ${SERVICE}: ${MSG}" >>$DEST + fi +} + +#Get the day delta since the archive file backup +seconds_difference() { + archive_date=$( date --date="$1" +%s ) + if [ "$?" -ne 0 ] + then + second_delta=0 + fi + current_date=$( date +%s ) + second_delta=$(($current_date-$archive_date)) + if [ "$second_delta" -lt 0 ] + then + second_delta=0 + fi + echo $second_delta +} + +# Wait for a file to be available on the file system (written by the other +# container). +wait_for_file() { + WAIT_FILE=$1 + NO_TIMEOUT=${2:-false} + TIMEOUT=300 + if [[ $NO_TIMEOUT == "true" ]] + then + # Such a large value to virtually never timeout + TIMEOUT=999999999 + fi + TIMEOUT_EXP=$(( $(date +%s) + $TIMEOUT )) + DONE=false + while [[ $DONE == "false" ]] + do + DELTA=$(( TIMEOUT_EXP - $(date +%s) )) + if [[ "$(ls -l ${WAIT_FILE} 2>/dev/null | wc -l)" -gt 0 ]]; + then + DONE=true + elif [[ $DELTA -lt 0 ]] + then + DONE=true + echo "Timed out waiting for file ${WAIT_FILE}." + return 1 + else + echo "Still waiting ...will time out in ${DELTA} seconds..." + sleep 5 + fi + done + return 0 +} + diff --git a/postgresql/templates/bin/_db_test.sh.tpl b/postgresql/templates/bin/_db_test.sh.tpl new file mode 100644 index 0000000000..8accacec11 --- /dev/null +++ b/postgresql/templates/bin/_db_test.sh.tpl @@ -0,0 +1,85 @@ +#!/bin/bash +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +trap cleanup EXIT SIGTERM SIGINT SIGKILL + +TEST_DATABASE_NAME="pg_helmtest_db" +TEST_DATABASE_USER="pg_helmtest_user" +TEST_DATABASE_PASSWORD=$RANDOM +TEST_TABLE_NAME="pg_helmtest" + +function psql_cmd { + DATABASE=$1 + DB_USER=$2 + export PGPASSWORD=$3 + DB_COMMAND=$4 + EXIT_ON_FAIL=${5:-1} + + psql \ + -h $DB_FQDN \ + -p $DB_PORT \ + -U $DB_USER \ + -d $DATABASE \ + -v "ON_ERROR_STOP=1" \ + --command="${DB_COMMAND}" + + RC=$? + + if [[ $RC -ne 0 ]] + then + echo 'FAIL!' + if [[ $EXIT_ON_FAIL -eq 1 ]] + then + exit $RC + fi + fi + + return 0 +} + +function cleanup { + echo 'Cleaning up the database...' + psql_cmd "postgres" ${DB_ADMIN_USER} ${ADMIN_PASSWORD} "DROP DATABASE IF EXISTS ${TEST_DATABASE_NAME};" 0 + psql_cmd "postgres" ${DB_ADMIN_USER} ${ADMIN_PASSWORD} "DROP ROLE IF EXISTS ${TEST_DATABASE_USER};" 0 + echo 'Cleanup Finished.' +} + +# Create db +echo 'Testing database connectivity as admin user...' +psql_cmd "postgres" ${DB_ADMIN_USER} ${ADMIN_PASSWORD} "SELECT 1 FROM pg_database;" +echo 'Connectivity Test SUCCESS!' + +echo 'Testing creation of an application database...' +psql_cmd "postgres" ${DB_ADMIN_USER} ${ADMIN_PASSWORD} "CREATE DATABASE ${TEST_DATABASE_NAME};" +echo 'Database Creation Test SUCCESS!' + +echo 'Testing creation of an application user...' +psql_cmd "postgres" ${DB_ADMIN_USER} ${ADMIN_PASSWORD} "CREATE ROLE ${TEST_DATABASE_USER} LOGIN PASSWORD '${TEST_DATABASE_PASSWORD}';" +psql_cmd "postgres" ${DB_ADMIN_USER} ${ADMIN_PASSWORD} "GRANT ALL PRIVILEGES ON DATABASE ${TEST_DATABASE_NAME} to ${TEST_DATABASE_USER};" +echo 'User Creation SUCCESS!' + +echo 'Testing creation of an application table...' +psql_cmd ${TEST_DATABASE_NAME} ${TEST_DATABASE_USER} ${TEST_DATABASE_PASSWORD} "CREATE TABLE ${TEST_TABLE_NAME} (name text);" +echo 'Table Creation SUCCESS!' + +echo 'Testing DML...' +psql_cmd ${TEST_DATABASE_NAME} ${TEST_DATABASE_USER} ${TEST_DATABASE_PASSWORD} "INSERT INTO ${TEST_TABLE_NAME} (name) VALUES ('test.');" +psql_cmd ${TEST_DATABASE_NAME} ${TEST_DATABASE_USER} ${TEST_DATABASE_PASSWORD} "SELECT * FROM ${TEST_TABLE_NAME};" +psql_cmd ${TEST_DATABASE_NAME} ${TEST_DATABASE_USER} ${TEST_DATABASE_PASSWORD} "DELETE FROM ${TEST_TABLE_NAME};" +echo 'DML Test SUCCESS!' + +exit 0 diff --git a/postgresql/templates/bin/_postgresql_archive_cleanup.sh.tpl b/postgresql/templates/bin/_postgresql_archive_cleanup.sh.tpl new file mode 100644 index 0000000000..d8ed7bb1b7 --- /dev/null +++ b/postgresql/templates/bin/_postgresql_archive_cleanup.sh.tpl @@ -0,0 +1,46 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set +ex + +# ARCHIVE_LIMIT env variable is Threshold of archiving supposed to be kept in percentage +clean_up () { + echo "Cleanup required as Utilization is above threshold" + # Get file count and delete half of the archive while maintaining the order of the files + FILE_COUNT=$(ls -1 ${ARCHIVE_PATH} | sort | wc -l) + COUNT=0 + echo $((FILE_COUNT/2)) + for file in $(ls -1 ${ARCHIVE_PATH} | sort); do + if [[ $COUNT -lt $((FILE_COUNT/2)) ]]; then + echo "removing following file $file" + rm -rf ${ARCHIVE_PATH}/$file + else + break + fi + COUNT=$((COUNT+1)) + done +} +#infinite loop to check the utilization of archive +while true +do + # checking the utilization of archive directory + UTILIZATION=$(df -h ${ARCHIVE_PATH} | awk ' NR==2 {print $5} ' | awk '{ print substr( $0, 1, length($0)-1 ) }') + if [[ $UTILIZATION -gt ${ARCHIVE_LIMIT} ]]; + then + clean_up + fi + sleep 3600 +done + + diff --git a/postgresql/templates/bin/_readiness.sh.tpl b/postgresql/templates/bin/_readiness.sh.tpl new file mode 100644 index 0000000000..8aefefddcf --- /dev/null +++ b/postgresql/templates/bin/_readiness.sh.tpl @@ -0,0 +1,19 @@ +#!/usr/bin/env bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +pg_isready -U ${POSTGRES_USER} diff --git a/postgresql/templates/bin/_remote_retrieve_postgresql.sh.tpl b/postgresql/templates/bin/_remote_retrieve_postgresql.sh.tpl new file mode 100755 index 0000000000..fc685b6124 --- /dev/null +++ b/postgresql/templates/bin/_remote_retrieve_postgresql.sh.tpl @@ -0,0 +1,81 @@ +#!/bin/bash + +# Copyright 2018 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -x + +RESTORE_DIR=${POSTGRESQL_BACKUP_BASE_DIR}/db/${POSTGRESQL_POD_NAMESPACE}/postgres/restore +ARCHIVE_DIR=${POSTGRESQL_BACKUP_BASE_DIR}/db/${POSTGRESQL_POD_NAMESPACE}/postgres/archive + +source /tmp/common_backup_restore.sh + +# Keep processing requests for the life of the pod. +while true +do + # Wait until a restore request file is present on the disk + echo "Waiting for a restore request..." + NO_TIMEOUT=true + wait_for_file $RESTORE_DIR/*_request $NO_TIMEOUT + + echo "Done waiting. Request received" + + CONTAINER_NAME={{ .Values.conf.backup.remote_backup.container_name }} + + if [[ -e $RESTORE_DIR/archive_listing_request ]] + then + # We've finished consuming the request, so delete the request file. + rm -rf $RESTORE_DIR/*_request + + openstack container show $CONTAINER_NAME + if [[ $? -eq 0 ]] + then + # Get the list, ensureing that we only pick up postgres backups from the + # requested namespace + openstack object list $CONTAINER_NAME | grep postgres | grep $POSTGRESQL_POD_NAMESPACE | awk '{print $2}' > $RESTORE_DIR/archive_list_response + if [[ $? != 0 ]] + then + echo "Container object listing could not be obtained." >> $RESTORE_DIR/archive_list_error + else + echo "Archive listing successfully retrieved." + fi + else + echo "Container $CONTAINER_NAME does not exist." >> $RESTORE_DIR/archive_list_error + fi + elif [[ -e $RESTORE_DIR/get_archive_request ]] + then + ARCHIVE=`cat $RESTORE_DIR/get_archive_request` + + echo "Request for archive $ARCHIVE received." + + # We've finished consuming the request, so delete the request file. + rm -rf $RESTORE_DIR/*_request + + openstack object save --file $RESTORE_DIR/$ARCHIVE $CONTAINER_NAME $ARCHIVE + if [[ $? != 0 ]] + then + echo "Archive $ARCHIVE could not be retrieved." >> $RESTORE_DIR/archive_error + else + echo "Archive $ARCHIVE successfully retrieved." + fi + + # Signal to the other container that the archive is available. + touch $RESTORE_DIR/archive_response + else + rm -rf $RESTORE_DIR/*_request + echo "Invalid request received." + fi + + sleep 5 +done diff --git a/postgresql/templates/bin/_remote_store_postgresql.sh.tpl b/postgresql/templates/bin/_remote_store_postgresql.sh.tpl new file mode 100755 index 0000000000..6eb2b3a134 --- /dev/null +++ b/postgresql/templates/bin/_remote_store_postgresql.sh.tpl @@ -0,0 +1,208 @@ +#!/bin/bash + +# Copyright 2018 The Openstack-Helm Authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Note: not using set -e because more elaborate error handling is required. +set -x + +BACKUPS_DIR=${POSTGRESQL_BACKUP_BASE_DIR}/db/${POSTGRESQL_POD_NAMESPACE}/postgres/current + +# Create the working backups directory if the other container didn't already, +# and if this container creates it first, ensure that permissions are writable +# for the other container (running as "postgres" user) in the same "postgres" +# group. +mkdir -p $BACKUPS_DIR || log_backup_error_exit "Cannot create directory ${BACKUPS_DIR}!" 1 +chmod 775 $BACKUPS_DIR + +source /tmp/common_backup_restore.sh + +#Send backup file to storage +send_to_storage() { + FILEPATH=$1 + FILE=$2 + + CONTAINER_NAME={{ .Values.conf.backup.remote_backup.container_name }} + + # Grab the list of containers on the remote site + RESULT=$(openstack container list 2>&1) + + if [[ $? == 0 ]] + then + echo $RESULT | grep $CONTAINER_NAME + if [[ $? != 0 ]] + then + # Create the container + openstack container create $CONTAINER_NAME || log ERROR postgresql_backup "Cannot create container ${CONTAINER_NAME}!" + openstack container show $CONTAINER_NAME + if [[ $? != 0 ]] + then + log ERROR postgresql_backup "Error retrieving container $CONTAINER_NAME after creation." + return 1 + fi + fi + else + echo $RESULT | grep "HTTP 401" + if [[ $? == 0 ]] + then + log ERROR postgresql_backup "Could not access keystone: HTTP 401" + return 1 + else + echo $RESULT | grep "ConnectionError" + if [[ $? == 0 ]] + then + log ERROR postgresql_backup "Could not access keystone: ConnectionError" + # In this case, keystone or the site/node may be temporarily down. + # Return slightly different error code so the calling code can retry + return 2 + else + log ERROR postgresql_backup "Could not get container list: ${RESULT}" + return 1 + fi + fi + fi + + # Create an object to store the file + openstack object create --name $FILE $CONTAINER_NAME $FILEPATH/$FILE || log ERROR postgresql_backup "Cannot create container object ${FILE}!" + openstack object show $CONTAINER_NAME $FILE + if [[ $? != 0 ]] + then + log ERROR postgresql_backup "Error retrieving container object $FILE after creation." + return 1 + fi + + log INFO postgresql_backup "Created file $FILE in container $CONTAINER_NAME successfully." + return 0 +} + +if {{ .Values.conf.backup.remote_backup.enabled }} +then + WAIT_FOR_BACKUP_TIMEOUT=1800 + WAIT_FOR_RGW_AVAIL_TIMEOUT=1800 + + # Wait until a backup file is ready to ship to RGW, or until we time out. + DONE=false + TIMEOUT_EXP=$(( $(date +%s) + $WAIT_FOR_BACKUP_TIMEOUT )) + while [[ $DONE == "false" ]] + do + log INFO postgresql_backup "Waiting for a backup file to be written to the disk." + sleep 5 + DELTA=$(( TIMEOUT_EXP - $(date +%s) )) + ls -l ${BACKUPS_DIR}/backup_completed + if [[ $? -eq 0 ]] + then + DONE=true + elif [[ $DELTA -lt 0 ]] + then + DONE=true + fi + done + + log INFO postgresql_backup "Done waiting." + FILE_TO_SEND=$(ls $BACKUPS_DIR/*.tar.gz) + + ERROR_SEEN=false + + if [[ $FILE_TO_SEND != "" ]] + then + if [[ $(echo $FILE_TO_SEND | wc -w) -gt 1 ]] + then + # There should only be one backup file to send - this is an error + log_backup_error_exit "More than one backup file found (${FILE_TO_SEND}) - can only handle 1!" 1 + fi + + # Get just the filename from the file (strip the path) + FILE=$(basename $FILE_TO_SEND) + + log INFO postgresql_backup "Backup file ${BACKUPS_DIR}/${FILE} found." + + DONE=false + TIMEOUT_EXP=$(( $(date +%s) + $WAIT_FOR_RGW_AVAIL_TIMEOUT )) + while [[ $DONE == "false" ]] + do + # Store the new archive to the remote backup storage facility. + send_to_storage $BACKUPS_DIR $FILE + + # Check if successful + if [[ $? -eq 0 ]] + then + log INFO postgresql_backup "Backup file ${BACKUPS_DIR}/${FILE} successfully sent to RGW. Deleting from current backup directory." + DONE=true + elif [[ $? -eq 2 ]] + then + # Temporary failure occurred. We need to retry if we haven't timed out + log WARN postgresql_backup "Backup file ${BACKUPS_DIR}/${FILE} could not be sent to RGW due to connection issue." + DELTA=$(( TIMEOUT_EXP - $(date +%s) )) + if [[ $DELTA -lt 0 ]] + then + DONE=true + log ERROR postgresql_backup "Timed out waiting for RGW to become available." + ERROR_SEEN=true + else + log INFO postgresql_backup "Sleeping 30 seconds waiting for RGW to become available..." + sleep 30 + log INFO postgresql_backup "Retrying..." + fi + else + log ERROR postgresql_backup "Backup file ${BACKUPS_DIR}/${FILE} could not be sent to the RGW." + ERROR_SEEN=true + DONE=true + fi + done + else + log ERROR postgresql_backup "No backup file found in $BACKUPS_DIR." + ERROR_SEEN=true + fi + + if [[ $ERROR_SEEN == "true" ]] + then + log ERROR postgresql_backup "Errors encountered. Exiting." + exit 1 + fi + + # At this point, we should remove the files in current dir. + # If an error occurred, then we need the file to remain there for future + # container restarts, and maybe it will eventually succeed. + rm -rf $BACKUPS_DIR/* + + #Only delete an old archive after a successful archive + if [ "${POSTGRESQL_BACKUP_DAYS_TO_KEEP}" -gt 0 ] + then + log INFO postgresql_backup "Deleting backups older than ${POSTGRESQL_BACKUP_DAYS_TO_KEEP} days" + BACKUP_FILES=/tmp/backup_files + PG_BACKUP_FILES=/tmp/pg_backup_files + + openstack object list $CONTAINER_NAME > $BACKUP_FILES + if [[ $? != 0 ]] + then + log_backup_error_exit "Could not obtain a list of current backup files in the RGW" 1 + fi + + # Filter out other types of files like mariadb, etcd backupes etc.. + cat $BACKUP_FILES | grep postgres | grep $POSTGRESQL_POD_NAMESPACE | awk '{print $2}' > $PG_BACKUP_FILES + + for ARCHIVE_FILE in $(cat $PG_BACKUP_FILES) + do + ARCHIVE_DATE=$( echo $ARCHIVE_FILE | awk -F/ '{print $NF}' | cut -d'.' -f 4) + if [ "$(seconds_difference ${ARCHIVE_DATE})" -gt "$((${POSTGRESQL_BACKUP_DAYS_TO_KEEP}*86400))" ] + then + log INFO postgresql_backup "Deleting file ${ARCHIVE_FILE} from the RGW" + openstack object delete $CONTAINER_NAME $ARCHIVE_FILE || log_backup_error_exit "Cannot delete container object ${ARCHIVE_FILE}!" 1 + fi + done + fi +else + log INFO postgresql_backup "Remote backup is not enabled" + exit 0 +fi diff --git a/postgresql/templates/bin/_restore_postgresql.sh.tpl b/postgresql/templates/bin/_restore_postgresql.sh.tpl new file mode 100755 index 0000000000..ed9702de3f --- /dev/null +++ b/postgresql/templates/bin/_restore_postgresql.sh.tpl @@ -0,0 +1,362 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +# Capture the user's command line arguments +ARGS=("$@") + +# This is needed to get the postgresql admin password +# Note: xtracing should be off so it doesn't print the pw +export PGPASSWORD=$(cat /etc/postgresql/admin_user.conf \ + | grep postgres | awk -F: '{print $5}') + +source /tmp/restore_main.sh + +# Export the variables needed by the framework +export DB_NAME="postgres" +export DB_NAMESPACE=${POSTGRESQL_POD_NAMESPACE} +export ARCHIVE_DIR=${POSTGRESQL_BACKUP_BASE_DIR}/db/${DB_NAMESPACE}/${DB_NAME}/archive + +# Define variables needed in this file +POSTGRESQL_HOST=$(cat /etc/postgresql/admin_user.conf | cut -d: -f 1) +export PSQL="psql -U $POSTGRESQL_ADMIN_USER -h $POSTGRESQL_HOST" +export LOG_FILE=/tmp/dbrestore.log + +# Extract all databases from an archive and put them in the requested +# file. +get_databases() { + TMP_DIR=$1 + DB_FILE=$2 + + SQL_FILE=$TMP_DIR/postgres.$POSTGRESQL_POD_NAMESPACE.*.sql + if [ -f $SQL_FILE ]; then + grep 'CREATE DATABASE' $SQL_FILE | awk '{ print $3 }' > $DB_FILE + else + # Error, cannot report the databases + echo "No SQL file found - cannot extract the databases" + return 1 + fi +} + +# Extract all tables of a database from an archive and put them in the requested +# file. +get_tables() { + DATABASE=$1 + TMP_DIR=$2 + TABLE_FILE=$3 + + SQL_FILE=$TMP_DIR/postgres.$POSTGRESQL_POD_NAMESPACE.*.sql + if [ -f $SQL_FILE ]; then + cat $SQL_FILE | sed -n /'\\connect '$DATABASE/,/'\\connect'/p | grep "CREATE TABLE" | awk -F'[. ]' '{print $4}' > $TABLE_FILE + else + # Error, cannot report the tables + echo "No SQL file found - cannot extract the tables" + return 1 + fi +} + +# Extract all rows in the given table of a database from an archive and put them in the requested +# file. +get_rows() { + DATABASE=$1 + TABLE=$2 + TMP_DIR=$3 + ROW_FILE=$4 + + SQL_FILE=$TMP_DIR/postgres.$POSTGRESQL_POD_NAMESPACE.*.sql + if [ -f $SQL_FILE ]; then + cat $SQL_FILE | sed -n /'\\connect '${DATABASE}/,/'\\connect'/p > /tmp/db.sql + cat /tmp/db.sql | grep "INSERT INTO public.${TABLE} VALUES" > $ROW_FILE + rm /tmp/db.sql + else + # Error, cannot report the rows + echo "No SQL file found - cannot extract the rows" + return 1 + fi +} + +# Extract the schema for the given table in the given database belonging to the archive file +# found in the TMP_DIR. +get_schema() { + DATABASE=$1 + TABLE=$2 + TMP_DIR=$3 + SCHEMA_FILE=$4 + + SQL_FILE=$TMP_DIR/postgres.$POSTGRESQL_POD_NAMESPACE.*.sql + if [ -f $SQL_FILE ]; then + DB_FILE=$(mktemp -p /tmp) + cat $SQL_FILE | sed -n /'\\connect '${DATABASE}/,/'\\connect'/p > ${DB_FILE} + cat ${DB_FILE} | sed -n /'CREATE TABLE public.'${TABLE}' ('/,/'--'/p > ${SCHEMA_FILE} + cat ${DB_FILE} | sed -n /'CREATE SEQUENCE public.'${TABLE}/,/'--'/p >> ${SCHEMA_FILE} + cat ${DB_FILE} | sed -n /'ALTER TABLE public.'${TABLE}/,/'--'/p >> ${SCHEMA_FILE} + cat ${DB_FILE} | sed -n /'ALTER TABLE ONLY public.'${TABLE}/,/'--'/p >> ${SCHEMA_FILE} + cat ${DB_FILE} | sed -n /'ALTER SEQUENCE public.'${TABLE}/,/'--'/p >> ${SCHEMA_FILE} + cat ${DB_FILE} | sed -n /'SELECT pg_catalog.*public.'${TABLE}/,/'--'/p >> ${SCHEMA_FILE} + cat ${DB_FILE} | sed -n /'CREATE INDEX.*public.'${TABLE}' USING'/,/'--'/p >> ${SCHEMA_FILE} + cat ${DB_FILE} | sed -n /'GRANT.*public.'${TABLE}' TO'/,/'--'/p >> ${SCHEMA_FILE} + rm -f ${DB_FILE} + else + # Error, cannot report the rows + echo "No SQL file found - cannot extract the schema" + return 1 + fi +} + +# Extract Single Database SQL Dump from pg_dumpall dump file +extract_single_db_dump() { + ARCHIVE=$1 + DATABASE=$2 + DIR=$3 + sed -n '/\\connect'" ${DATABASE}/,/PostgreSQL database dump complete/p" ${ARCHIVE} > ${DIR}/${DATABASE}.sql +} + +# Re-enable connections to a database +reenable_connections() { + SINGLE_DB_NAME=$1 + + # First make sure this is not the main postgres database or either of the + # two template databases that should not be touched. + if [[ ${SINGLE_DB_NAME} == "postgres" || + ${SINGLE_DB_NAME} == "template0" || + ${SINGLE_DB_NAME} == "template1" ]]; then + echo "Cannot re-enable connections on an postgres internal db ${SINGLE_DB_NAME}" + return 1 + fi + + # Re-enable connections to the DB + $PSQL -tc "UPDATE pg_database SET datallowconn = 'true' WHERE datname = '${SINGLE_DB_NAME}';" > /dev/null 2>&1 + if [[ "$?" -ne 0 ]]; then + echo "Could not re-enable connections for database ${SINGLE_DB_NAME}" + return 1 + fi + return 0 +} + +# Drop connections from a database +drop_connections() { + SINGLE_DB_NAME=$1 + + # First make sure this is not the main postgres database or either of the + # two template databases that should not be touched. + if [[ ${SINGLE_DB_NAME} == "postgres" || + ${SINGLE_DB_NAME} == "template0" || + ${SINGLE_DB_NAME} == "template1" ]]; then + echo "Cannot drop connections on an postgres internal db ${SINGLE_DB_NAME}" + return 1 + fi + + # First, prevent any new connections from happening on this database. + $PSQL -tc "UPDATE pg_database SET datallowconn = 'false' WHERE datname = '${SINGLE_DB_NAME}';" > /dev/null 2>&1 + if [[ "$?" -ne 0 ]]; then + echo "Could not prevent new connections before restoring database ${SINGLE_DB_NAME}." + return 1 + fi + + # Next, force disconnection of all clients currently connected to this database. + $PSQL -tc "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE datname = '${SINGLE_DB_NAME}';" > /dev/null 2>&1 + if [[ "$?" -ne 0 ]]; then + echo "Could not drop existing connections before restoring database ${SINGLE_DB_NAME}." + reenable_connections ${SINGLE_DB_NAME} + return 1 + fi + return 0 +} + +# Re-enable connections for all of the databases within Postgresql +reenable_connections_on_all_dbs() { + # Get a list of the databases + DB_LIST=$($PSQL -tc "\l" | grep "| postgres |" | awk '{print $1}') + + RET=0 + + # Re-enable the connections for each of the databases. + for DB in $DB_LIST; do + if [[ ${DB} != "postgres" && ${DB} != "template0" && ${DB} != "template1" ]]; then + reenable_connections $DB + if [[ "$?" -ne 0 ]]; then + RET=1 + fi + fi + done + + return $RET +} + +# Drop connections in all of the databases within Postgresql +drop_connections_on_all_dbs() { + # Get a list of the databases + DB_LIST=$($PSQL -tc "\l" | grep "| postgres |" | awk '{print $1}') + + RET=0 + + # Drop the connections for each of the databases. + for DB in $DB_LIST; do + # Make sure this is not the main postgres database or either of the + # two template databases that should not be touched. + if [[ ${DB} != "postgres" && ${DB} != "template0" && ${DB} != "template1" ]]; then + drop_connections $DB + if [[ "$?" -ne 0 ]]; then + RET=1 + fi + fi + done + + # If there was a failure to drop any connections, go ahead and re-enable + # them all to prevent a lock-out condition + if [[ $RET -ne 0 ]]; then + reenable_connections_on_all_dbs + fi + + return $RET +} + +# Restore a single database dump from pg_dumpall sql dumpfile. +restore_single_db() { + SINGLE_DB_NAME=$1 + TMP_DIR=$2 + + # Reset the logfile incase there was some older log there + rm -rf ${LOG_FILE} + touch ${LOG_FILE} + + # First make sure this is not the main postgres database or either of the + # two template databases that should not be touched. + if [[ ${SINGLE_DB_NAME} == "postgres" || + ${SINGLE_DB_NAME} == "template0" || + ${SINGLE_DB_NAME} == "template1" ]]; then + echo "Cannot restore an postgres internal db ${SINGLE_DB_NAME}" + return 1 + fi + + SQL_FILE=$TMP_DIR/postgres.$POSTGRESQL_POD_NAMESPACE.*.sql + if [ -f $SQL_FILE ]; then + extract_single_db_dump $SQL_FILE $SINGLE_DB_NAME $TMP_DIR + if [[ -f $TMP_DIR/$SINGLE_DB_NAME.sql && -s $TMP_DIR/$SINGLE_DB_NAME.sql ]]; then + # Drop connections first + drop_connections ${SINGLE_DB_NAME} + if [[ "$?" -ne 0 ]]; then + return 1 + fi + + # Next, drop the database + $PSQL -tc "DROP DATABASE $SINGLE_DB_NAME;" + if [[ "$?" -ne 0 ]]; then + echo "Could not drop the old ${SINGLE_DB_NAME} database before restoring it." + reenable_connections ${SINGLE_DB_NAME} + return 1 + fi + + # Postgresql does not have the concept of creating database if condition. + # This next command creates the database in case it does not exist. + $PSQL -tc "SELECT 1 FROM pg_database WHERE datname = '$SINGLE_DB_NAME'" | grep -q 1 || \ + $PSQL -c "CREATE DATABASE $SINGLE_DB_NAME" + if [[ "$?" -ne 0 ]]; then + echo "Could not create the single database being restored: ${SINGLE_DB_NAME}." + reenable_connections ${SINGLE_DB_NAME} + return 1 + fi + $PSQL -d $SINGLE_DB_NAME -f ${TMP_DIR}/${SINGLE_DB_NAME}.sql 2>>$LOG_FILE >> $LOG_FILE + if [[ "$?" -eq 0 ]]; then + if grep "ERROR:" ${LOG_FILE} > /dev/null 2>&1; then + cat $LOG_FILE + echo "Errors occurred during the restore of database ${SINGLE_DB_NAME}" + reenable_connections ${SINGLE_DB_NAME} + return 1 + else + echo "Database restore Successful." + fi + else + # Dump out the log file for debugging + cat $LOG_FILE + echo -e "\nDatabase restore Failed." + reenable_connections ${SINGLE_DB_NAME} + return 1 + fi + + # Re-enable connections to the DB + reenable_connections ${SINGLE_DB_NAME} + if [[ "$?" -ne 0 ]]; then + return 1 + fi + else + echo "Database dump For $SINGLE_DB_NAME is empty or not available." + return 1 + fi + else + echo "No database file available to restore from." + return 1 + fi + return 0 +} + +# Restore all the databases from the pg_dumpall sql file. +restore_all_dbs() { + TMP_DIR=$1 + + # Reset the logfile incase there was some older log there + rm -rf ${LOG_FILE} + touch ${LOG_FILE} + + SQL_FILE=$TMP_DIR/postgres.$POSTGRESQL_POD_NAMESPACE.*.sql + if [ -f $SQL_FILE ]; then + + # Check the scope of the archive. + SCOPE=$(echo ${SQL_FILE} | awk -F'.' '{print $(NF-1)}') + if [[ "${SCOPE}" != "all" ]]; then + # This is just a single database backup. The user should + # instead use the single database restore option. + echo "Cannot use the restore all option for an archive containing only a single database." + echo "Please use the single database restore option." + return 1 + fi + + # First drop all connections on all databases + drop_connections_on_all_dbs + if [[ "$?" -ne 0 ]]; then + return 1 + fi + + $PSQL postgres -f $SQL_FILE 2>>$LOG_FILE >> $LOG_FILE + if [[ "$?" -eq 0 ]]; then + if grep "ERROR:" ${LOG_FILE} > /dev/null 2>&1; then + cat ${LOG_FILE} + echo "Errors occurred during the restore of the databases." + reenable_connections_on_all_dbs + return 1 + else + echo "Database Restore Successful." + fi + else + # Dump out the log file for debugging + cat ${LOG_FILE} + echo -e "\nDatabase Restore failed." + reenable_connections_on_all_dbs + return 1 + fi + + # Re-enable connections on all databases + reenable_connections_on_all_dbs + if [[ "$?" -ne 0 ]]; then + return 1 + fi + else + echo "There is no database file available to restore from." + return 1 + fi + return 0 +} + +# Call the CLI interpreter, providing the archive directory path and the +# user arguments passed in +cli_main ${ARGS[@]} diff --git a/postgresql/templates/bin/_start.sh.tpl b/postgresql/templates/bin/_start.sh.tpl new file mode 100644 index 0000000000..db49a1deab --- /dev/null +++ b/postgresql/templates/bin/_start.sh.tpl @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# Disable echo mode while setting the password +# unless we are in debug mode +{{- if .Values.conf.debug }} +set -x +{{- end }} +set -e + +POSTGRES_DB=${POSTGRES_DB:-"postgres"} + +# Check if the Postgres data directory exists before attempting to +# set the password +if [[ -d "$PGDATA" && -s "$PGDATA/PG_VERSION" ]] +then + postgres --single -D "$PGDATA" "$POSTGRES_DB" < /var/lib/archive/%f' + cluster_name: 'postgresql' + datestyle: 'iso, mdy' + external_pid_file: '/tmp/postgres.pid' + fsync: 'on' + listen_addresses: '0.0.0.0' + log_checkpoints: 'on' + log_connections: 'on' + log_disconnections: 'on' + log_line_prefix: 'postgresql: %t [%p]: [%l-1] %c %x %d %u %a %h %m ' + log_lock_waits: 'on' + log_temp_files: '0' + log_timezone: 'UTC' + max_connections: '1000' + max_locks_per_transaction: '64' + max_prepared_transactions: '0' + max_wal_senders: '16' + max_worker_processes: '10' + port: '5432' + shared_buffers: '2GB' + ssl: 'off' + ssl_cert_file: '/server_certs/tls.crt' + ssl_ca_file: '/server_certs/ca.crt' + ssl_key_file: '/server_certs/tls.key' + ssl_ciphers: 'TLSv1.2:!aNULL' + tcp_keepalives_idle: '900' + tcp_keepalives_interval: '100' + timezone: 'UTC' + track_commit_timestamp: 'on' + track_functions: 'all' + wal_keep_size: '256' + wal_level: 'hot_standby' + wal_log_hints: 'on' + hba_file: '/tmp/pg_hba.conf' + ident_file: '/tmp/pg_ident.conf' + backup: + enabled: false + base_path: /var/backup + days_to_keep: 3 + pg_dumpall_options: '--inserts --clean' + remote_backup: + enabled: false + container_name: postgresql + days_to_keep: 14 + storage_policy: default-placement + number_of_retries: 5 + delay_range: + min: 30 + max: 60 + throttle_backups: + enabled: false + sessions_limit: 480 + lock_expire_after: 7200 + retry_after: 3600 + container_name: throttle-backups-manager + + exporter: + queries: + pg_postmaster: + query: "SELECT pg_postmaster_start_time as start_time_seconds from pg_postmaster_start_time()" + master: true + metrics: + - start_time_seconds: + usage: "GAUGE" + description: "Time at which postmaster started" + +secrets: + oci_image_registry: + postgresql: postgresql-oci-image-registry-key + postgresql: + admin: postgresql-admin + exporter: postgresql-exporter + audit: postgresql-audit + backup_restore: postgresql-backup-restore + tls: + server: + internal: postgresql-tls-direct + identity: + admin: keystone-admin-user + postgresql: postgresql-backup-user + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + postresql: + username: postresql + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + postgresql: + auth: + admin: + username: postgres + password: password + exporter: + username: psql_exporter + password: psql_exp_pass + audit: + username: audit + password: password + hosts: + default: postgresql + host_fqdn_override: + default: null + path: null + scheme: postgresql + port: + postgresql: + default: 5432 + postgresql_restapi: + hosts: + default: postgresql-restapi + host_fqdn_override: + default: null + path: null + scheme: postgresql + port: + restapi: + default: 8008 + prometheus_postgresql_exporter: + namespace: null + hosts: + default: postgresql-exporter + host_fqdn_override: + default: null + path: + default: /metrics + scheme: + default: 'http' + port: + metrics: + default: 9187 + identity: + name: backup-storage-auth + namespace: openstack + auth: + admin: + # Auth URL of null indicates local authentication + # HTK will form the URL unless specified here + auth_url: null + region_name: RegionOne + username: admin + password: password + project_name: admin + user_domain_name: default + project_domain_name: default + postgresql: + # Auth URL of null indicates local authentication + # HTK will form the URL unless specified here + auth_url: null + role: admin + region_name: RegionOne + username: postgresql-backup-user + password: password + project_name: service + user_domain_name: service + project_domain_name: service + hosts: + default: keystone + internal: keystone-api + host_fqdn_override: + default: null + path: + default: /v3 + scheme: + default: 'http' + port: + api: + default: 80 + internal: 5000 + +manifests: + certificates: false + configmap_bin: true + configmap_etc: true + job_image_repo_sync: true + network_policy: false + job_ks_user: false + secret_admin: true + secret_etc: true + secret_audit: true + secret_backup_restore: false + secret_registry: true + service: true + statefulset: true + cron_job_postgresql_backup: false + pvc_backup: false + monitoring: + prometheus: + configmap_bin: true + configmap_etc: true + deployment_exporter: true + job_user_create: true + secret_etc: true + service_exporter: true +... diff --git a/powerdns/Chart.yaml b/powerdns/Chart.yaml new file mode 100644 index 0000000000..0b2f5e9ec7 --- /dev/null +++ b/powerdns/Chart.yaml @@ -0,0 +1,26 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: v4.1.10 +description: OpenStack-Helm PowerDNS +name: powerdns +version: 2024.2.0 +home: https://www.powerdns.com/ +maintainers: + - name: OpenStack-Helm Authors +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit + version: ">= 0.1.0" +... diff --git a/powerdns/templates/bin/_powerdns-mysql-sync.sh.tpl b/powerdns/templates/bin/_powerdns-mysql-sync.sh.tpl new file mode 100644 index 0000000000..c8d8c56387 --- /dev/null +++ b/powerdns/templates/bin/_powerdns-mysql-sync.sh.tpl @@ -0,0 +1,22 @@ +#!/bin/sh + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +MYSQLCMD='mysql -r -N' +if [ $(echo 'show tables' | $MYSQLCMD | wc -c) -eq 0 ]; then + $MYSQLCMD < /etc/pdns/schema.sql +fi diff --git a/powerdns/templates/configmap-bin.yaml b/powerdns/templates/configmap-bin.yaml new file mode 100644 index 0000000000..cbbbee81b0 --- /dev/null +++ b/powerdns/templates/configmap-bin.yaml @@ -0,0 +1,31 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_bin }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: powerdns-bin +data: +{{- if .Values.images.local_registry.active }} + image-repo-sync.sh: | +{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} +{{- end }} + db-init.py: | +{{- include "helm-toolkit.scripts.db_init" . | indent 4 }} + powerdns-mysql-sync.sh: | +{{ tuple "bin/_powerdns-mysql-sync.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} diff --git a/powerdns/templates/configmap-etc.yaml b/powerdns/templates/configmap-etc.yaml new file mode 100644 index 0000000000..88901c8da0 --- /dev/null +++ b/powerdns/templates/configmap-etc.yaml @@ -0,0 +1,56 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "powerdns.configmap.etc" -}} +{{- range $key, $value := . }} +{{ $key | replace "_" "-" }} = {{ $value }} +{{- end }} +{{- end -}} + +{{- if .Values.manifests.configmap_etc }} +{{- $mysql := .Values.conf.mysql.client }} + +{{- if empty $mysql.host -}} +{{- $_ := tuple "oslo_db" "internal" . | include "helm-toolkit.endpoints.endpoint_host_lookup" | set $mysql "host" -}} +{{- $_ := $mysql.host | set .Values.conf.powerdns "gmysql_host" -}} +{{- end -}} + +{{- if empty $mysql.port -}} +{{- $_ := tuple "oslo_db" "internal" "mysql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | set $mysql "port" -}} +{{- $_ := $mysql.port | set .Values.conf.powerdns "gmysql_port" -}} +{{- end -}} + +{{- if empty $mysql.user -}} +{{- $_ := .Values.endpoints.oslo_db.auth.powerdns.username | set $mysql "user" -}} +{{- $_ := $mysql.user | set .Values.conf.powerdns "gmysql_user" -}} +{{- end -}} + +{{- if empty $mysql.password -}} +{{- $_ := .Values.endpoints.oslo_db.auth.powerdns.password | set $mysql "password" -}} +{{- $_ := $mysql.password | set .Values.conf.powerdns "gmysql_password" -}} +{{- end -}} + +{{- if empty .Values.conf.powerdns.api_key -}} +{{- $_ := tuple "powerdns" "service" . | include "helm-toolkit.endpoints.endpoint_token_lookup" | set .Values.conf.powerdns "api_key" -}} +{{- end -}} +--- +apiVersion: v1 +kind: Secret +metadata: + name: powerdns-etc +type: Opaque +data: + pdns.conf: {{ include "powerdns.configmap.etc" .Values.conf.powerdns | b64enc }} + my.cnf: {{ include "helm-toolkit.utils.to_ini" .Values.conf.mysql | b64enc }} +{{- end }} diff --git a/powerdns/templates/deployment.yaml b/powerdns/templates/deployment.yaml new file mode 100644 index 0000000000..319395156b --- /dev/null +++ b/powerdns/templates/deployment.yaml @@ -0,0 +1,77 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.deployment }} +{{- $envAll := . }} + +{{- $serviceAccountName := "powerdns" }} +{{ tuple $envAll "powerdns" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: powerdns + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "powerdns" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + replicas: {{ .Values.pod.replicas.server }} + selector: + matchLabels: +{{ tuple $envAll "powerdns" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} +{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "powerdns" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + spec: + serviceAccountName: {{ $serviceAccountName }} + affinity: +{{ tuple $envAll "powerdns" "server" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} + nodeSelector: + {{ .Values.labels.powerdns.node_selector_key }}: {{ .Values.labels.powerdns.node_selector_value | quote }} + initContainers: +{{ tuple $envAll "powerdns" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: powerdns +{{ tuple $envAll "powerdns" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + command: + - pdns_server + ports: + - containerPort: {{ tuple "powerdns" "internal" "powerdns" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + name: pdns-udp + protocol: UDP + - containerPort: {{ tuple "powerdns" "internal" "powerdns_tcp" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + name: pdns-tcp + - containerPort: {{ tuple "powerdns" "internal" "powerdns_api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + name: pdns-api + readinessProbe: + tcpSocket: + port: {{ tuple "powerdns" "internal" "powerdns_tcp" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + volumeMounts: + - name: powerdns-etc + mountPath: /etc/pdns/conf.d/pdns.conf + subPath: pdns.conf + readOnly: true + volumes: + - name: powerdns-etc + secret: + secretName: powerdns-etc + defaultMode: 0444 +{{- end }} diff --git a/powerdns/templates/job-db-init.yaml b/powerdns/templates/job-db-init.yaml new file mode 100644 index 0000000000..c2f2531f7e --- /dev/null +++ b/powerdns/templates/job-db-init.yaml @@ -0,0 +1,21 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.job_db_init }} + +{{- $dbToInit := dict "inputType" "secret" "adminSecret" .Values.secrets.oslo_db.admin "userSecret" .Values.secrets.oslo_db.powerdns -}} +{{- $dbInitJob := dict "envAll" . "serviceName" "powerdns" "dbToInit" $dbToInit -}} +{{ $dbInitJob | include "helm-toolkit.manifests.job_db_init_mysql" }} + +{{- end }} diff --git a/powerdns/templates/job-db-sync.yaml b/powerdns/templates/job-db-sync.yaml new file mode 100644 index 0000000000..ff29640768 --- /dev/null +++ b/powerdns/templates/job-db-sync.yaml @@ -0,0 +1,64 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.job_db_sync }} +{{- $envAll := . }} + + +{{- $serviceAccountName := "powerdns-db-sync" }} +{{ tuple $envAll "db_sync" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ $serviceAccountName }} + labels: +{{ tuple $envAll "powerdns" "db-sync" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + template: + metadata: + labels: +{{ tuple $envAll "powerdns" "db-sync" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + spec: + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value }} + initContainers: +{{ tuple $envAll "db_sync" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: powerdns-db-sync +{{ tuple $envAll "db_sync" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.db_sync | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + command: + - /tmp/powerdns-mysql-sync.sh + volumeMounts: + - name: powerdns-bin + mountPath: /tmp/powerdns-mysql-sync.sh + subPath: powerdns-mysql-sync.sh + readOnly: true + - name: powerdns-etc + mountPath: /etc/mysql/my.cnf + subPath: my.cnf + readOnly: true + volumes: + - name: powerdns-bin + configMap: + name: powerdns-bin + defaultMode: 0555 + - name: powerdns-etc + secret: + secretName: powerdns-etc + defaultMode: 0444 +{{- end }} diff --git a/powerdns/templates/job-image-repo-sync.yaml b/powerdns/templates/job-image-repo-sync.yaml new file mode 100644 index 0000000000..2f9aadedc0 --- /dev/null +++ b/powerdns/templates/job-image-repo-sync.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "powerdns" -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/powerdns/templates/secret-db.yaml b/powerdns/templates/secret-db.yaml new file mode 100644 index 0000000000..2da122fa09 --- /dev/null +++ b/powerdns/templates/secret-db.yaml @@ -0,0 +1,28 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_db }} +{{- $envAll := . }} +{{- range $key1, $userClass := tuple "admin" "powerdns" }} +{{- $secretName := index $envAll.Values.secrets.oslo_db $userClass }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} +type: Opaque +data: + DB_CONNECTION: {{ tuple "oslo_db" "internal" $userClass "mysql" $envAll | include "helm-toolkit.endpoints.authenticated_endpoint_uri_lookup" | b64enc -}} +{{- end }} +{{- end }} diff --git a/powerdns/templates/secret-registry.yaml b/powerdns/templates/secret-registry.yaml new file mode 100644 index 0000000000..da979b3223 --- /dev/null +++ b/powerdns/templates/secret-registry.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_registry .Values.endpoints.oci_image_registry.auth.enabled }} +{{ include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) }} +{{- end }} diff --git a/powerdns/templates/service.yaml b/powerdns/templates/service.yaml new file mode 100644 index 0000000000..4ea8485011 --- /dev/null +++ b/powerdns/templates/service.yaml @@ -0,0 +1,45 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.service_dns }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "powerdns" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +spec: + ports: + - port: {{ tuple "powerdns" "internal" "powerdns" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + name: pdns-udp + protocol: UDP + - port: {{ tuple "powerdns" "internal" "powerdns_tcp" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + name: pdns-tcp + {{- if .Values.manifests.service_api }} + - port: {{ tuple "powerdns" "internal" "powerdns_api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + name: pdns-api + {{- end }} + selector: +{{ tuple $envAll "powerdns" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + {{- if .Values.network.node_port_enabled }} +{{/* +Set Type=NodePort to get output packets from cluster internal IP +of the POD instead of container one. +*/}} + type: NodePort + {{- if .Values.network.external_policy_local }} + externalTrafficPolicy: Local + {{- end }} + {{- end }} +{{- end }} diff --git a/powerdns/values.yaml b/powerdns/values.yaml new file mode 100644 index 0000000000..e5d5d3756a --- /dev/null +++ b/powerdns/values.yaml @@ -0,0 +1,222 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for powerdns. +# This is a YAML-formatted file. +# Declare name/value pairs to be passed into your templates. +# name: value + +--- +images: + tags: + powerdns: docker.io/psitrax/powerdns:4.1.10 + db_init: docker.io/openstackhelm/heat:wallaby-ubuntu_focal + db_sync: docker.io/psitrax/powerdns:4.1.10 + dep_check: quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal + image_repo_sync: docker.io/library/docker:17.07.0 + pull_policy: IfNotPresent + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +pod: + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + weight: + default: 10 + replicas: + server: 1 + lifecycle: + upgrades: + deployments: + revision_history: 3 + pod_replacement_strategy: RollingUpdate + rolling_update: + max_unavailable: 1 + max_surge: 3 + resources: + enabled: false + server: + limits: + memory: "128Mi" + cpu: "500m" + requests: + memory: "128Mi" + cpu: "500m" + jobs: + image_repo_sync: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + tests: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + +labels: + powerdns: + node_selector_key: openstack-control-plane + node_selector_value: enabled + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + test: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - powerdns-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + powerdns: + jobs: + - powerdns-db-init + - powerdns-db-sync + services: + - endpoint: internal + service: oslo_db + db_init: + services: + - endpoint: internal + service: oslo_db + db_sync: + jobs: + - powerdns-db-init + services: + - service: oslo_db + endpoint: internal + +network: + node_port_enabled: true + external_policy_local: true + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + powerdns: + username: powerdns + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + powerdns: + auth: + service: + token: chiave_segreta + hosts: + default: powerdns + host_fqdn_override: + default: null + port: + powerdns_api: + default: 8081 + powerdns_tcp: + default: 53 + powerdns: + default: 53 + protocol: UDP + oslo_db: + auth: + admin: + username: root + password: password + powerdns: + username: powerdns + password: password + hosts: + default: mariadb + host_fqdn_override: + default: null + path: /powerdns + scheme: mysql+pymysql + port: + mysql: + default: 3306 + +secrets: + oci_image_registry: + powerdns: powerdns-oci-image-registry-key + oslo_db: + admin: powerdns-db-admin + powerdns: powerdns-db-user + +conf: + powerdns: + slave: true + dnsupdate: true + api: true + cache_ttl: 0 + query_cache_ttl: 0 + negquery_cache_ttl: 0 + out_of_zone_additional_processing: no + webserver: true + webserver_address: 0.0.0.0 + webserver_allow_from: 0.0.0.0/0 + gmysql_dbname: powerdns + gmysql_dnssec: yes + mysql: + client: + database: powerdns + +manifests: + configmap_bin: true + configmap_etc: true + deployment: true + job_db_init: true + job_db_sync: true + secret_db: true + secret_registry: true + service_dns: true + service_api: false +... diff --git a/prometheus-alertmanager/Chart.yaml b/prometheus-alertmanager/Chart.yaml new file mode 100644 index 0000000000..467af7e2b3 --- /dev/null +++ b/prometheus-alertmanager/Chart.yaml @@ -0,0 +1,29 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: v0.20.0 +description: OpenStack-Helm Alertmanager for Prometheus +name: prometheus-alertmanager +version: 2024.2.0 +home: https://prometheus.io/docs/alerting/alertmanager/ +sources: + - https://github.com/prometheus/alertmanager + - https://opendev.org/openstack/openstack-helm-infra +maintainers: + - name: OpenStack-Helm Authors +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit + version: ">= 0.1.0" +... diff --git a/prometheus-alertmanager/templates/bin/_alertmanager.sh.tpl b/prometheus-alertmanager/templates/bin/_alertmanager.sh.tpl new file mode 100644 index 0000000000..1838a05ca2 --- /dev/null +++ b/prometheus-alertmanager/templates/bin/_alertmanager.sh.tpl @@ -0,0 +1,42 @@ +#!/bin/sh + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +COMMAND="${@:-start}" + +function start () { + exec /bin/alertmanager \ + --config.file=/etc/alertmanager/config.yml \ +{{- range $flag, $value := .Values.conf.command_flags.alertmanager }} +{{- $flag := $flag | replace "_" "-" }} +{{ printf "--%s=%s" $flag $value | indent 4 }} \ +{{- end }} + $(generate_peers) +} + +function generate_peers () { + final_pod_suffix=$(( {{ .Values.pod.replicas.alertmanager }}-1 )) + for pod_suffix in `seq 0 "$final_pod_suffix"` + do + echo --cluster.peer=prometheus-alertmanager-$pod_suffix.$DISCOVERY_SVC:$MESH_PORT + done +} + +function stop () { + kill -TERM 1 +} + +$COMMAND diff --git a/prometheus-alertmanager/templates/bin/_apache.sh.tpl b/prometheus-alertmanager/templates/bin/_apache.sh.tpl new file mode 100644 index 0000000000..f2f55dacda --- /dev/null +++ b/prometheus-alertmanager/templates/bin/_apache.sh.tpl @@ -0,0 +1,44 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -exv + +COMMAND="${@:-start}" + +function start () { + + if [ -f /etc/apache2/envvars ]; then + # Loading Apache2 ENV variables + source /etc/httpd/apache2/envvars + fi + # Apache gets grumpy about PID files pre-existing + rm -f /etc/httpd/logs/httpd.pid + + if [ -f /usr/local/apache2/conf/.htpasswd ]; then + htpasswd -b /usr/local/apache2/conf/.htpasswd "$ALERTMANAGER_USERNAME" "$ALERTMANAGER_PASSWORD" + else + htpasswd -cb /usr/local/apache2/conf/.htpasswd "$ALERTMANAGER_USERNAME" "$ALERTMANAGER_PASSWORD" + fi + + #Launch Apache on Foreground + exec httpd -DFOREGROUND +} + +function stop () { + apachectl -k graceful-stop +} + +$COMMAND diff --git a/prometheus-alertmanager/templates/clusterrolebinding.yaml b/prometheus-alertmanager/templates/clusterrolebinding.yaml new file mode 100644 index 0000000000..cb50866322 --- /dev/null +++ b/prometheus-alertmanager/templates/clusterrolebinding.yaml @@ -0,0 +1,31 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.clusterrolebinding }} +{{- $envAll := . }} +{{- $serviceAccountName := printf "%s" "prometheus-alertmanager" }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: run-alertmanager +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +roleRef: + kind: ClusterRole + name: cluster-admin + apiGroup: rbac.authorization.k8s.io +{{- end }} diff --git a/prometheus-alertmanager/templates/configmap-bin.yaml b/prometheus-alertmanager/templates/configmap-bin.yaml new file mode 100644 index 0000000000..63abf91f54 --- /dev/null +++ b/prometheus-alertmanager/templates/configmap-bin.yaml @@ -0,0 +1,29 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_bin }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ printf "%s-%s" $envAll.Release.Name "alertmanager-bin" | quote }} +data: + apache.sh: | +{{ tuple "bin/_apache.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + alertmanager.sh: | +{{ tuple "bin/_alertmanager.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + image-repo-sync.sh: | +{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} +{{- end }} diff --git a/prometheus-alertmanager/templates/configmap-etc.yaml b/prometheus-alertmanager/templates/configmap-etc.yaml new file mode 100644 index 0000000000..b7a1f4ef4a --- /dev/null +++ b/prometheus-alertmanager/templates/configmap-etc.yaml @@ -0,0 +1,28 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_etc }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ printf "%s-%s" $envAll.Release.Name "alertmanager-etc" | quote }} +data: +{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.alertmanager "key" "config.yml" "format" "Secret") | indent 2 }} +{{- if .Values.conf.alert_templates }} +{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.alert_templates "key" "alert-templates.tmpl" "format" "Secret") | indent 2 }} +{{- end }} +{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.httpd "key" "httpd.conf" "format" "Secret") | indent 2 }} +{{- end }} diff --git a/prometheus-alertmanager/templates/ingress-alertmanager.yaml b/prometheus-alertmanager/templates/ingress-alertmanager.yaml new file mode 100644 index 0000000000..bd4475bf63 --- /dev/null +++ b/prometheus-alertmanager/templates/ingress-alertmanager.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.ingress .Values.network.alertmanager.ingress.public }} +{{- $ingressOpts := dict "envAll" . "backendService" "alertmanager" "backendServiceType" "alertmanager" "backendPort" "http" -}} +{{ $ingressOpts | include "helm-toolkit.manifests.ingress" }} +{{- end }} diff --git a/prometheus-alertmanager/templates/job-image-repo-sync.yaml b/prometheus-alertmanager/templates/job-image-repo-sync.yaml new file mode 100644 index 0000000000..1294d2522e --- /dev/null +++ b/prometheus-alertmanager/templates/job-image-repo-sync.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "alertmanager" -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/prometheus-alertmanager/templates/network_policy.yaml b/prometheus-alertmanager/templates/network_policy.yaml new file mode 100644 index 0000000000..2f87afb4ae --- /dev/null +++ b/prometheus-alertmanager/templates/network_policy.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */}} + +{{- if .Values.manifests.network_policy -}} +{{- $netpol_opts := dict "envAll" . "name" "application" "label" "alertmanager" -}} +{{ $netpol_opts | include "helm-toolkit.manifests.kubernetes_network_policy" }} +{{- end -}} diff --git a/prometheus-alertmanager/templates/secret-admin-user.yaml b/prometheus-alertmanager/templates/secret-admin-user.yaml new file mode 100644 index 0000000000..a80f856471 --- /dev/null +++ b/prometheus-alertmanager/templates/secret-admin-user.yaml @@ -0,0 +1,26 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_admin_user }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ printf "%s-%s" $envAll.Release.Name "admin-user" | quote }} +type: Opaque +data: + ALERTMANAGER_USERNAME: {{ .Values.endpoints.alertmanager.auth.admin.username | b64enc }} + ALERTMANAGER_PASSWORD: {{ .Values.endpoints.alertmanager.auth.admin.password | b64enc }} +{{- end }} diff --git a/prometheus-alertmanager/templates/secret-ingress-tls.yaml b/prometheus-alertmanager/templates/secret-ingress-tls.yaml new file mode 100644 index 0000000000..e3b8b79a51 --- /dev/null +++ b/prometheus-alertmanager/templates/secret-ingress-tls.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_ingress_tls }} +{{- include "helm-toolkit.manifests.secret_ingress_tls" ( dict "envAll" . "backendServiceType" "alertmanager" "backendService" "alertmanager") }} +{{- end }} diff --git a/prometheus-alertmanager/templates/secret-registry.yaml b/prometheus-alertmanager/templates/secret-registry.yaml new file mode 100644 index 0000000000..da979b3223 --- /dev/null +++ b/prometheus-alertmanager/templates/secret-registry.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_registry .Values.endpoints.oci_image_registry.auth.enabled }} +{{ include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) }} +{{- end }} diff --git a/prometheus-alertmanager/templates/service-discovery.yaml b/prometheus-alertmanager/templates/service-discovery.yaml new file mode 100644 index 0000000000..8d63e82c28 --- /dev/null +++ b/prometheus-alertmanager/templates/service-discovery.yaml @@ -0,0 +1,30 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.service_discovery }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "alertmanager" "discovery" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +spec: + type: ClusterIP + clusterIP: None + ports: + - name: peer-mesh + port: {{ tuple "alertmanager" "internal" "mesh" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + selector: +{{ tuple $envAll "prometheus-alertmanager" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- end }} diff --git a/prometheus-alertmanager/templates/service-ingress-alertmanager.yaml b/prometheus-alertmanager/templates/service-ingress-alertmanager.yaml new file mode 100644 index 0000000000..8e33e420a0 --- /dev/null +++ b/prometheus-alertmanager/templates/service-ingress-alertmanager.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.service_ingress .Values.network.alertmanager.ingress.public }} +{{- $serviceIngressOpts := dict "envAll" . "backendServiceType" "alertmanager" -}} +{{ $serviceIngressOpts | include "helm-toolkit.manifests.service_ingress" }} +{{- end }} diff --git a/prometheus-alertmanager/templates/service.yaml b/prometheus-alertmanager/templates/service.yaml new file mode 100644 index 0000000000..03c50b9129 --- /dev/null +++ b/prometheus-alertmanager/templates/service.yaml @@ -0,0 +1,39 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.service }} +{{- $envAll := . }} +{{- $prometheus_annotations := $envAll.Values.monitoring.prometheus.prometheus }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "alertmanager" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + annotations: +{{- if .Values.monitoring.prometheus.enabled }} +{{ tuple $prometheus_annotations | include "helm-toolkit.snippets.prometheus_service_annotations" | indent 4 }} +{{- end }} +spec: + ports: + - name: http + port: {{ tuple "alertmanager" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + {{ if .Values.network.alertmanager.node_port.enabled }} + nodePort: {{ .Values.network.alertmanager.node_port.port }} + {{ end }} + selector: +{{ tuple $envAll "prometheus-alertmanager" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + {{ if .Values.network.alertmanager.node_port.enabled }} + type: NodePort + {{ end }} +{{- end }} diff --git a/prometheus-alertmanager/templates/statefulset.yaml b/prometheus-alertmanager/templates/statefulset.yaml new file mode 100644 index 0000000000..453eec153c --- /dev/null +++ b/prometheus-alertmanager/templates/statefulset.yaml @@ -0,0 +1,186 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.statefulset }} +{{- $envAll := . }} + +{{- $mounts_alertmanager := .Values.pod.mounts.alertmanager.alertmanager }} +{{- $mounts_alertmanager_init := .Values.pod.mounts.alertmanager.init_container }} + +{{- $serviceAccountName := "prometheus-alertmanager" }} +{{ tuple $envAll "prometheus-alertmanager" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: prometheus-alertmanager + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "prometheus-alertmanager" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + serviceName: {{ tuple "alertmanager" "discovery" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + podManagementPolicy: "Parallel" + replicas: {{ .Values.pod.replicas.alertmanager }} + selector: + matchLabels: +{{ tuple $envAll "prometheus-alertmanager" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} + template: + metadata: + labels: +{{ tuple $envAll "prometheus-alertmanager" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} +{{ dict "envAll" $envAll "podName" $serviceAccountName "containerNames" (list "prometheus-alertmanager" "prometheus-alertmanager-perms" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "server" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + affinity: +{{ tuple $envAll "prometheus-alertmanager" "server" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} + nodeSelector: + {{ .Values.labels.alertmanager.node_selector_key }}: {{ .Values.labels.alertmanager.node_selector_value | quote }} + terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.alertmanager.timeout | default "30" }} + initContainers: +{{ tuple $envAll "alertmanager" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + - name: prometheus-alertmanager-perms +{{ tuple $envAll "prometheus-alertmanager" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.alertmanager | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "server" "container" "prometheus_alertmanager_perms" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - chown + - -R + - "nobody:" + - /var/lib/alertmanager/data + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: alertmanager-data + mountPath: /var/lib/alertmanager/data + containers: + - name: apache-proxy +{{ tuple $envAll "apache_proxy" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.apache_proxy | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "server" "container" "apache_proxy" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/apache.sh + - start + ports: + - name: http + containerPort: 80 + env: + - name: ALERTMANAGER_PORT + value: {{ tuple "alertmanager" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: ALERTMANAGER_USERNAME + valueFrom: + secretKeyRef: + name: {{ printf "%s-%s" $envAll.Release.Name "admin-user" | quote }} + key: ALERTMANAGER_USERNAME + - name: ALERTMANAGER_PASSWORD + valueFrom: + secretKeyRef: + name: {{ printf "%s-%s" $envAll.Release.Name "admin-user" | quote }} + key: ALERTMANAGER_PASSWORD + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: alertmanager-bin + mountPath: /tmp/apache.sh + subPath: apache.sh + readOnly: true + - name: alertmanager-etc + mountPath: /usr/local/apache2/conf/httpd.conf + subPath: httpd.conf + readOnly: true + - name: prometheus-alertmanager +{{ tuple $envAll "prometheus-alertmanager" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.alertmanager | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "server" "container" "prometheus_alertmanager" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/alertmanager.sh + - start + lifecycle: + preStop: + exec: + command: + - /tmp/alertmanager.sh + - stop + env: + - name: DISCOVERY_SVC + value: {{ tuple "alertmanager" "discovery" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }} + - name: MESH_PORT + value: {{ tuple "alertmanager" "internal" "mesh" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + ports: + - name: alerts-api + containerPort: {{ tuple "alertmanager" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - name: peer-mesh + containerPort: {{ tuple "alertmanager" "internal" "mesh" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + readinessProbe: + httpGet: + path: /#/status + port: {{ tuple "alertmanager" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + initialDelaySeconds: 30 + timeoutSeconds: 30 + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: etc-alertmanager + mountPath: /etc/config + {{- if .Values.conf.alert_templates }} + - name: alertmanager-etc + mountPath: /etc/alertmanager/template/alert-templates.tmpl + subPath: alert-templates.tmpl + readOnly: true + {{- end }} + - name: alertmanager-etc + mountPath: /etc/alertmanager/config.yml + subPath: config.yml + readOnly: true + - name: alertmanager-bin + mountPath: /tmp/alertmanager.sh + subPath: alertmanager.sh + readOnly: true + - name: alertmanager-data + mountPath: /var/lib/alertmanager/data +{{ if $mounts_alertmanager.volumeMounts }}{{ toYaml $mounts_alertmanager.volumeMounts | indent 12 }}{{ end }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: etc-alertmanager + emptyDir: {} + - name: alertmanager-etc + secret: + secretName: {{ printf "%s-%s" $envAll.Release.Name "alertmanager-etc" | quote }} + defaultMode: 0444 + - name: alertmanager-bin + configMap: + name: {{ printf "%s-%s" $envAll.Release.Name "alertmanager-bin" | quote }} + defaultMode: 0555 +{{ if $mounts_alertmanager.volumes }}{{ toYaml $mounts_alertmanager.volumes | indent 8 }}{{ end }} +{{- if not .Values.storage.alertmanager.enabled }} + - name: alertmanager-data + emptyDir: {} +{{- else }} + volumeClaimTemplates: + - metadata: + name: alertmanager-data + spec: + accessModes: {{ .Values.storage.alertmanager.pvc.access_mode }} + resources: + requests: + storage: {{ .Values.storage.alertmanager.requests.storage }} + storageClassName: {{ .Values.storage.alertmanager.storage_class }} +{{- end }} +{{- end }} diff --git a/prometheus-alertmanager/values.yaml b/prometheus-alertmanager/values.yaml new file mode 100644 index 0000000000..148230363f --- /dev/null +++ b/prometheus-alertmanager/values.yaml @@ -0,0 +1,480 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for alertmanager. +# This is a YAML-formatted file. +# Declare name/value pairs to be passed into your templates. +# name: value + +--- +images: + tags: + apache_proxy: docker.io/library/httpd:2.4 + prometheus-alertmanager: docker.io/prom/alertmanager:v0.20.0 + dep_check: quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal + image_repo_sync: docker.io/library/docker:17.07.0 + pull_policy: IfNotPresent + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +labels: + alertmanager: + node_selector_key: openstack-control-plane + node_selector_value: enabled + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +pod: + security_context: + server: + pod: + runAsUser: 65534 + container: + prometheus_alertmanager_perms: + runAsUser: 0 + readOnlyRootFilesystem: true + apache_proxy: + runAsUser: 0 + readOnlyRootFilesystem: false + prometheus_alertmanager: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + weight: + default: 10 + mounts: + alertmanager: + alertmanager: + init_container: null + replicas: + alertmanager: 1 + lifecycle: + upgrades: + deployment: + pod_replacement_strategy: RollingUpdate + statefulsets: + pod_replacement_strategy: RollingUpdate + termination_grace_period: + alertmanager: + timeout: 30 + resources: + enabled: false + apache_proxy: + limits: + memory: "1024Mi" + cpu: "2000m" + requests: + memory: "128Mi" + cpu: "100m" + alertmanager: + limits: + memory: "1024Mi" + cpu: "2000m" + requests: + memory: "128Mi" + cpu: "500m" + jobs: + image_repo_sync: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + prometheus-alertmanager: + username: prometheus-alertmanager + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + alertmanager: + name: prometheus-alertmanager + namespace: null + auth: + admin: + username: admin + password: changeme + hosts: + default: alerts-engine + public: prometheus-alertmanager + discovery: prometheus-alertmanager-discovery + host_fqdn_override: + default: null + # NOTE(srwilkers): this chart supports TLS for fqdn over-ridden public + # endpoints using the following format: + # public: + # host: null + # tls: + # crt: null + # key: null + path: + default: null + scheme: + default: 'http' + port: + api: + default: 9093 + public: 80 + mesh: + default: 9094 + http: + default: 80 + ldap: + hosts: + default: ldap + auth: + admin: + bind: "cn=admin,dc=cluster,dc=local" + password: password + host_fqdn_override: + default: null + path: + default: "/ou=People,dc=cluster,dc=local" + scheme: + default: ldap + port: + ldap: + default: 389 + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - alertmanager-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + alertmanager: + services: null + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + +network: + alertmanager: + ingress: + public: true + classes: + namespace: "nginx" + cluster: "nginx-cluster" + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / + node_port: + enabled: false + port: 30903 + +secrets: + oci_image_registry: + prometheus-alertmanager: prometheus-alertmanager-oci-image-registry-key + tls: + alertmanager: + alertmanager: + public: alerts-tls-public + +storage: + alertmanager: + enabled: true + pvc: + access_mode: ["ReadWriteOnce"] + requests: + storage: 5Gi + storage_class: general + +manifests: + clusterrolebinding: true + configmap_bin: true + configmap_etc: true + ingress: true + job_image_repo_sync: true + network_policy: false + secret_admin_user: true + secret_ingress_tls: true + secret_registry: true + service: true + service_discovery: true + service_ingress: true + statefulset: true + +network_policy: + alertmanager: + ingress: + - {} + egress: + - {} + +monitoring: + prometheus: + enabled: true + prometheus: + scrape: true + +conf: + httpd: | + ServerRoot "/usr/local/apache2" + + Listen 80 + + LoadModule mpm_event_module modules/mod_mpm_event.so + LoadModule authn_file_module modules/mod_authn_file.so + LoadModule authn_core_module modules/mod_authn_core.so + LoadModule authz_host_module modules/mod_authz_host.so + LoadModule authz_groupfile_module modules/mod_authz_groupfile.so + LoadModule authz_user_module modules/mod_authz_user.so + LoadModule authz_core_module modules/mod_authz_core.so + LoadModule access_compat_module modules/mod_access_compat.so + LoadModule auth_basic_module modules/mod_auth_basic.so + LoadModule ldap_module modules/mod_ldap.so + LoadModule authnz_ldap_module modules/mod_authnz_ldap.so + LoadModule reqtimeout_module modules/mod_reqtimeout.so + LoadModule filter_module modules/mod_filter.so + LoadModule proxy_html_module modules/mod_proxy_html.so + LoadModule log_config_module modules/mod_log_config.so + LoadModule env_module modules/mod_env.so + LoadModule headers_module modules/mod_headers.so + LoadModule setenvif_module modules/mod_setenvif.so + LoadModule version_module modules/mod_version.so + LoadModule proxy_module modules/mod_proxy.so + LoadModule proxy_connect_module modules/mod_proxy_connect.so + LoadModule proxy_http_module modules/mod_proxy_http.so + LoadModule proxy_balancer_module modules/mod_proxy_balancer.so + LoadModule remoteip_module modules/mod_remoteip.so + LoadModule slotmem_shm_module modules/mod_slotmem_shm.so + LoadModule slotmem_plain_module modules/mod_slotmem_plain.so + LoadModule unixd_module modules/mod_unixd.so + LoadModule status_module modules/mod_status.so + LoadModule autoindex_module modules/mod_autoindex.so + + + User daemon + Group daemon + + + + AllowOverride none + Require all denied + + + + Require all denied + + + ErrorLog /dev/stderr + + LogLevel warn + + + LogFormat "%a %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined + LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" proxy + LogFormat "%h %l %u %t \"%r\" %>s %b" common + + + LogFormat "%a %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combinedio + + + SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded + CustomLog /dev/stdout common + CustomLog /dev/stdout combined + CustomLog /dev/stdout proxy env=forwarded + + + + AllowOverride None + Options None + Require all granted + + + + RequestHeader unset Proxy early + + + + Include conf/extra/proxy-html.conf + + + + RemoteIPHeader X-Original-Forwarded-For + + ProxyPass http://localhost:{{ tuple "alertmanager" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/ + ProxyPassReverse http://localhost:{{ tuple "alertmanager" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/ + + + AuthName "Alertmanager" + AuthType Basic + AuthBasicProvider file ldap + AuthUserFile /usr/local/apache2/conf/.htpasswd + AuthLDAPBindDN {{ .Values.endpoints.ldap.auth.admin.bind }} + AuthLDAPBindPassword {{ .Values.endpoints.ldap.auth.admin.password }} + AuthLDAPURL {{ tuple "ldap" "default" "ldap" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | quote }} + Require valid-user + + + command_flags: + alertmanager: + storage.path: /var/lib/alertmanager/data + cluster.listen_address: "0.0.0.0:9094" + alertmanager: | + global: + # The smarthost and SMTP sender used for mail notifications. + smtp_smarthost: 'localhost:25' + smtp_from: 'alertmanager@example.org' + smtp_auth_username: 'alertmanager' + smtp_auth_password: 'password' + # The auth token for Hipchat. + hipchat_auth_token: '1234556789' + # Alternative host for Hipchat. + hipchat_api_url: 'https://hipchat.foobar.org/' + # The directory from which notification templates are read. + templates: + - '/etc/alertmanager/template/*.tmpl' + # The root route on which each incoming alert enters. + route: + # The labels by which incoming alerts are grouped together. For example, + # multiple alerts coming in for cluster=A and alertname=LatencyHigh would + # be batched into a single group. + group_by: + - alertname + - cluster + - service + # When a new group of alerts is created by an incoming alert, wait at + # least 'group_wait' to send the initial notification. + # This way ensures that you get multiple alerts for the same group that start + # firing shortly after another are batched together on the first + # notification. + group_wait: 30s + # When the first notification was sent, wait 'group_interval' to send a batch + # of new alerts that started firing for that group. + group_interval: 5m + # If an alert has successfully been sent, wait 'repeat_interval' to + # resend them. + repeat_interval: 3h + # A default receiver + # receiver: team-X-mails + receiver: 'team-X-mails' + # All the above attributes are inherited by all child routes and can + # overwritten on each. + # The child route trees. + routes: + # This routes performs a regular expression match on alert + # labels to catch alerts that are related to a list of + # services. + - receiver: 'team-X-mails' + continue: true + - match_re: + service: ^(foo1|foo2|baz)$ + receiver: team-X-mails + # The service has a sub-route for critical alerts, any alerts + # that do not match, i.e. severity != critical, fall-back to the + # parent node and are sent to 'team-X-mails' + routes: + - match: + severity: critical + receiver: team-X-pager + - match: + service: files + receiver: team-Y-mails + routes: + - match: + severity: critical + receiver: team-Y-pager + # This route handles all alerts coming from a database service. If there's + # no team to handle it, it defaults to the DB team. + - match: + service: database + receiver: team-DB-pager + # Also group alerts by affected database. + group_by: + - alertname + - cluster + - database + routes: + - match: + owner: team-X + receiver: team-X-pager + - match: + owner: team-Y + receiver: team-Y-pager + # Inhibition rules allow to mute a set of alerts given that another alert is + # firing. + # We use this to mute any warning-level notifications if the same alert is + # already critical. + inhibit_rules: + - source_match: + severity: 'critical' + target_match: + severity: 'warning' + # Apply inhibition if the alertname is the same. + equal: + - alertname + - cluster + - service + receivers: + - name: 'team-X-mails' + email_configs: + - to: 'team-X+alerts@example.org' + - name: 'team-X-pager' + email_configs: + - to: 'team-X+alerts-critical@example.org' + pagerduty_configs: + - service_key: + - name: 'team-Y-mails' + email_configs: + - to: 'team-Y+alerts@example.org' + - name: 'team-Y-pager' + pagerduty_configs: + - service_key: + - name: 'team-DB-pager' + pagerduty_configs: + - service_key: + - name: 'team-X-hipchat' + hipchat_configs: + - auth_token: + room_id: 85 + message_format: html + notify: false + alert_templates: null +... diff --git a/prometheus-blackbox-exporter/Chart.yaml b/prometheus-blackbox-exporter/Chart.yaml new file mode 100644 index 0000000000..5560cff9f5 --- /dev/null +++ b/prometheus-blackbox-exporter/Chart.yaml @@ -0,0 +1,28 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +apiVersion: v2 +appVersion: v0.16.0 +description: OpenStack-Helm blackbox exporter for Prometheus +name: prometheus-blackbox-exporter +version: 2024.2.0 +home: https://github.com/prometheus/blackbox_exporter +sources: + - https://opendev.org/openstack/openstack-helm-infra + - https://github.com/prometheus/blackbox_exporter +maintainers: + - name: OpenStack-Helm Authors +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit + version: ">= 0.1.0" +... diff --git a/prometheus-blackbox-exporter/templates/deployment.yaml b/prometheus-blackbox-exporter/templates/deployment.yaml new file mode 100644 index 0000000000..1845de0734 --- /dev/null +++ b/prometheus-blackbox-exporter/templates/deployment.yaml @@ -0,0 +1,69 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{- $envAll := . }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus-blackbox-exporter + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "prometheus-blackbox-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + replicas: {{ .Values.pod.replicas.prometheus_blackbox_exporter }} + selector: + matchLabels: +{{ tuple $envAll "prometheus-blackbox-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} +{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "prometheus-blackbox-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ dict "envAll" $envAll "podName" "prometheus-blackbox-exporter" "containerNames" (list "blackbox-exporter") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "prometheus_blackbox_exporter" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + nodeSelector: + {{ .Values.labels.blackbox_exporter.node_selector_key }}: {{ .Values.labels.blackbox_exporter.node_selector_value | quote }} + containers: + - name: blackbox-exporter +{{ tuple $envAll "blackbox_exporter" | include "helm-toolkit.snippets.image" | indent 8 }} +{{ tuple $envAll $envAll.Values.pod.resources.prometheus_blackbox_exporter | include "helm-toolkit.snippets.kubernetes_resources" | indent 8 }} +{{ dict "envAll" $envAll "application" "prometheus_blackbox_exporter" "container" "blackbox_exporter" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 8 }} + args: + - "--config.file=/config/blackbox.yaml" + ports: + - name: metrics + containerPort: {{ tuple "prometheus_blackbox_exporter" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + livenessProbe: + httpGet: + path: /health + port: {{ tuple "prometheus_blackbox_exporter" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + initialDelaySeconds: 30 + periodSeconds: 30 + readinessProbe: + httpGet: + path: /health + port: {{ tuple "prometheus_blackbox_exporter" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + initialDelaySeconds: 20 + periodSeconds: 30 + volumeMounts: + - mountPath: /config/blackbox.yaml + name: config + subPath: blackbox.yaml + volumes: + - name: config + secret: + secretName: prometheus-blackbox-exporter-etc diff --git a/prometheus-blackbox-exporter/templates/secret-registry.yaml b/prometheus-blackbox-exporter/templates/secret-registry.yaml new file mode 100644 index 0000000000..da979b3223 --- /dev/null +++ b/prometheus-blackbox-exporter/templates/secret-registry.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_registry .Values.endpoints.oci_image_registry.auth.enabled }} +{{ include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) }} +{{- end }} diff --git a/prometheus-blackbox-exporter/templates/secret.yaml b/prometheus-blackbox-exporter/templates/secret.yaml new file mode 100644 index 0000000000..9eba5ced36 --- /dev/null +++ b/prometheus-blackbox-exporter/templates/secret.yaml @@ -0,0 +1,23 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{- $envAll := . }} + +apiVersion: v1 +kind: Secret +metadata: + name: prometheus-blackbox-exporter-etc + labels: +{{ tuple $envAll "prometheus-blackbox-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +data: +{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.config.blackbox "key" "blackbox.yaml" "format" "Secret") | indent 2 }} diff --git a/prometheus-blackbox-exporter/templates/service.yaml b/prometheus-blackbox-exporter/templates/service.yaml new file mode 100644 index 0000000000..8eb8ef9f19 --- /dev/null +++ b/prometheus-blackbox-exporter/templates/service.yaml @@ -0,0 +1,26 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- $envAll := . }} + +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "prometheus_blackbox_exporter" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +spec: + ports: + - name: metrics + port: {{ tuple "prometheus_blackbox_exporter" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + selector: +{{ tuple $envAll "prometheus-blackbox-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} diff --git a/prometheus-blackbox-exporter/values.yaml b/prometheus-blackbox-exporter/values.yaml new file mode 100644 index 0000000000..80eb75dd23 --- /dev/null +++ b/prometheus-blackbox-exporter/values.yaml @@ -0,0 +1,143 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for kube-state-metrics. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. +--- + +images: + tags: + blackbox_exporter: docker.io/prom/blackbox-exporter:v0.16.0 + pull_policy: IfNotPresent + local_registry: + active: false +labels: + blackbox_exporter: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +service: + annotations: {} + port: 9115 + +secrets: + oci_image_registry: + prometheus-blackbox-exporter: prometheus-blackbox-exporter-oci-image-registry-key + +endpoints: + cluster_domain_suffix: cluster.local + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + prometheus-blackbox-exporter: + username: prometheus-blackbox-exporter + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + prometheus_blackbox_exporter: + namespace: null + hosts: + default: prometheus-blackbox-exporter + host_fqdn_override: + default: null + path: + default: null + scheme: + default: 'http' + port: + metrics: + default: 9115 + +pod: + security_context: + prometheus_blackbox_exporter: + pod: + runAsUser: 65534 + container: + blackbox_exporter: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + replicas: + prometheus_blackbox_exporter: 1 + annotations: + prometheus.io/scrape: 'true' + prometheus.io/port: "9115" + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + + lifecycle: + upgrades: + deployments: + revision_history: 3 + pod_replacement_strategy: RollingUpdate + rolling_update: + max_unavailable: 1 + max_surge: 3 + termination_grace_period: + prometheus_blackbox_exporter: + timeout: 30 + resources: + enabled: true + prometheus_blackbox_exporter: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - prometheus-openstack-exporter-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + prometheus_blackbox_exporter: + jobs: + - prometheus-openstack-exporter-ks-user + services: + - endpoint: internal + service: identity + +config: + blackbox: + modules: + http_2xx: + prober: http + timeout: 10s + http: + valid_http_versions: ["HTTP/1.1", "HTTP/2.0"] + no_follow_redirects: false + preferred_ip_protocol: "ip4" + +manifests: + secret_registry: true +... diff --git a/prometheus-kube-state-metrics/Chart.yaml b/prometheus-kube-state-metrics/Chart.yaml new file mode 100644 index 0000000000..24ec9df9a3 --- /dev/null +++ b/prometheus-kube-state-metrics/Chart.yaml @@ -0,0 +1,29 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: v1.3.1 +description: OpenStack-Helm Kube-State-Metrics for Prometheus +name: prometheus-kube-state-metrics +version: 2024.2.0 +home: https://github.com/kubernetes/kube-state-metrics +sources: + - https://github.com/kubernetes/kube-state-metrics + - https://opendev.org/openstack/openstack-helm-infra +maintainers: + - name: OpenStack-Helm Authors +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit + version: ">= 0.1.0" +... diff --git a/prometheus-kube-state-metrics/templates/configmap-bin.yaml b/prometheus-kube-state-metrics/templates/configmap-bin.yaml new file mode 100644 index 0000000000..9cdd5bbcf5 --- /dev/null +++ b/prometheus-kube-state-metrics/templates/configmap-bin.yaml @@ -0,0 +1,25 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_bin }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: kube-state-metrics-bin +data: + image-repo-sync.sh: | +{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} +{{- end }} diff --git a/prometheus-kube-state-metrics/templates/deployment.yaml b/prometheus-kube-state-metrics/templates/deployment.yaml new file mode 100644 index 0000000000..d4cf729661 --- /dev/null +++ b/prometheus-kube-state-metrics/templates/deployment.yaml @@ -0,0 +1,100 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "kubeMetricsReadinessProbe" }} +httpGet: + path: /metrics + port: {{ tuple "kube_state_metrics" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +{{- end }} + +{{- if .Values.manifests.deployment }} +{{- $envAll := . }} + +{{- $serviceAccountName := printf "%s-%s" .Release.Name "kube-state-metrics" }} +{{ tuple $envAll "kube_state_metrics" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ $serviceAccountName }} +rules: + - apiGroups: + - "*" + resources: + - "*" + verbs: + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ $serviceAccountName }} +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ $envAll.Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ $serviceAccountName }} + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: kube-state-metrics + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "kube-state-metrics" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + replicas: {{ .Values.pod.replicas.kube_state_metrics }} + selector: + matchLabels: +{{ tuple $envAll "kube-state-metrics" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} +{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "kube-state-metrics" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} +{{ dict "envAll" $envAll "podName" "kube-state-metrics" "containerNames" (list "kube-state-metrics" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "exporter" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + affinity: +{{ tuple $envAll "kube-state-metrics" "exporter" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} + nodeSelector: + {{ .Values.labels.kube_state_metrics.node_selector_key }}: {{ .Values.labels.kube_state_metrics.node_selector_value | quote }} + terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.kube_state_metrics.timeout | default "30" }} + initContainers: +{{ tuple $envAll "kube_state_metrics" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: kube-state-metrics +{{ tuple $envAll "kube_state_metrics" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.kube_state_metrics | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "exporter" "container" "kube_state_metrics" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + ports: + - name: metrics + containerPort: {{ tuple "kube_state_metrics" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +{{ dict "envAll" . "component" "server" "container" "kube_metrics" "type" "readiness" "probeTemplate" (include "kubeMetricsReadinessProbe" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }} + volumeMounts: + - name: pod-tmp + mountPath: /tmp + volumes: + - name: pod-tmp + emptyDir: {} +{{- end }} diff --git a/prometheus-kube-state-metrics/templates/job-image-repo-sync.yaml b/prometheus-kube-state-metrics/templates/job-image-repo-sync.yaml new file mode 100644 index 0000000000..a1e985a189 --- /dev/null +++ b/prometheus-kube-state-metrics/templates/job-image-repo-sync.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "kube-state-metrics" -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/prometheus-kube-state-metrics/templates/network_policy.yaml b/prometheus-kube-state-metrics/templates/network_policy.yaml new file mode 100644 index 0000000000..b8bbe583bd --- /dev/null +++ b/prometheus-kube-state-metrics/templates/network_policy.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */}} + +{{- if .Values.manifests.network_policy -}} +{{- $netpol_opts := dict "envAll" . "name" "application" "label" "kube-state-metrics" -}} +{{ $netpol_opts | include "helm-toolkit.manifests.kubernetes_network_policy" }} +{{- end -}} diff --git a/prometheus-kube-state-metrics/templates/secret-registry.yaml b/prometheus-kube-state-metrics/templates/secret-registry.yaml new file mode 100644 index 0000000000..da979b3223 --- /dev/null +++ b/prometheus-kube-state-metrics/templates/secret-registry.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_registry .Values.endpoints.oci_image_registry.auth.enabled }} +{{ include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) }} +{{- end }} diff --git a/prometheus-kube-state-metrics/templates/service-controller-manager.yaml b/prometheus-kube-state-metrics/templates/service-controller-manager.yaml new file mode 100644 index 0000000000..e60934e0be --- /dev/null +++ b/prometheus-kube-state-metrics/templates/service-controller-manager.yaml @@ -0,0 +1,39 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.service_controller_manager }} +{{- $envAll := . }} +{{- $prometheus_annotations := $envAll.Values.monitoring.prometheus.kube_controller_manager }} +--- +apiVersion: v1 +kind: Service +metadata: + name: kube-controller-manager-discovery + labels: +{{ tuple $envAll "controller-manager" "metrics" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: +{{- if .Values.monitoring.prometheus.enabled }} +{{ tuple $prometheus_annotations | include "helm-toolkit.snippets.prometheus_service_annotations" | indent 4 }} +{{- end }} +spec: + selector: + component: kube-controller-manager + type: ClusterIP + clusterIP: None + ports: + - name: http-metrics + port: {{ tuple "kube_controller_manager" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + targetPort: {{ tuple "kube_controller_manager" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + protocol: TCP +{{- end }} diff --git a/prometheus-kube-state-metrics/templates/service-kube-state-metrics.yaml b/prometheus-kube-state-metrics/templates/service-kube-state-metrics.yaml new file mode 100644 index 0000000000..bb52d60026 --- /dev/null +++ b/prometheus-kube-state-metrics/templates/service-kube-state-metrics.yaml @@ -0,0 +1,36 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.service_kube_state_metrics }} +{{- $envAll := . }} +{{- $prometheus_annotations := $envAll.Values.monitoring.prometheus.kube_state_metrics }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "kube_state_metrics" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + labels: +{{ tuple $envAll "kube-state-metrics" "metrics" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: +{{- if .Values.monitoring.prometheus.enabled }} +{{ tuple $prometheus_annotations | include "helm-toolkit.snippets.prometheus_service_annotations" | indent 4 }} +{{- end }} +spec: + ports: + - name: http + port: {{ tuple "kube_state_metrics" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + targetPort: {{ tuple "kube_state_metrics" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + selector: +{{ tuple $envAll "kube-state-metrics" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- end }} diff --git a/prometheus-kube-state-metrics/templates/service-scheduler.yaml b/prometheus-kube-state-metrics/templates/service-scheduler.yaml new file mode 100644 index 0000000000..ec5690ee90 --- /dev/null +++ b/prometheus-kube-state-metrics/templates/service-scheduler.yaml @@ -0,0 +1,39 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.service_scheduler }} +{{- $envAll := . }} +{{- $prometheus_annotations := $envAll.Values.monitoring.prometheus.kube_scheduler }} +--- +apiVersion: v1 +kind: Service +metadata: + name: kube-scheduler-discovery + labels: +{{ tuple $envAll "kube-scheduler" "metrics" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: +{{- if .Values.monitoring.prometheus.enabled }} +{{ tuple $prometheus_annotations | include "helm-toolkit.snippets.prometheus_service_annotations" | indent 4 }} +{{- end }} +spec: + selector: + component: kube-scheduler + type: ClusterIP + clusterIP: None + ports: + - name: http-metrics + port: {{ tuple "kube_scheduler" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + targetPort: {{ tuple "kube_scheduler" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + protocol: TCP +{{- end }} diff --git a/prometheus-kube-state-metrics/values.yaml b/prometheus-kube-state-metrics/values.yaml new file mode 100644 index 0000000000..a8fe82c1b0 --- /dev/null +++ b/prometheus-kube-state-metrics/values.yaml @@ -0,0 +1,206 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for kube-state-metrics. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +--- +images: + tags: + kube_state_metrics: quay.io/coreos/kube-state-metrics:v2.0.0-alpha.1 + dep_check: quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal + image_repo_sync: docker.io/library/docker:17.07.0 + pull_policy: IfNotPresent + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +labels: + kube_state_metrics: + node_selector_key: openstack-control-plane + node_selector_value: enabled + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +pod: + probes: + server: + kube_metrics: + readiness: + enabled: true + params: + initialDelaySeconds: 30 + periodSeconds: 60 + timeoutSeconds: 10 + security_context: + exporter: + pod: + runAsUser: 65534 + container: + kube_state_metrics: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + weight: + default: 10 + mounts: + kube_state_metrics: + kube_state_metrics: + init_container: null + replicas: + kube_state_metrics: 1 + lifecycle: + upgrades: + deployments: + revision_history: 3 + pod_replacement_strategy: RollingUpdate + rolling_update: + max_unavailable: 1 + max_surge: 3 + termination_grace_period: + kube_state_metrics: + timeout: 30 + resources: + enabled: false + kube_state_metrics: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + jobs: + image_repo_sync: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - kube-metrics-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + kube_state_metrics: + services: null + +secrets: + oci_image_registry: + prometheus-kube-state-metrics: prometheus-kube-state-metrics-oci-image-registry-key + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + prometheus-kube-state-metrics: + username: prometheus-kube-state-metrics + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + kube_state_metrics: + namespace: null + hosts: + default: kube-state-metrics + host_fqdn_override: + default: null + path: + default: null + scheme: + default: 'http' + port: + http: + default: 8080 + kube_scheduler: + scheme: + default: 'http' + path: + default: /metrics + port: + metrics: + default: 10251 + kube_controller_manager: + scheme: + default: 'http' + path: + default: /metrics + port: + metrics: + default: 10252 + +network_policy: + kube-state-metrics: + ingress: + - {} + egress: + - {} + +monitoring: + prometheus: + enabled: true + kube_state_metrics: + scrape: true + kube_scheduler: + scrape: true + kube_controller_manager: + scrape: true + +manifests: + configmap_bin: true + deployment: true + job_image_repo_sync: true + network_policy: false + secret_registry: true + service_kube_state_metrics: true + service_controller_manager: true + service_scheduler: true + serviceaccount: true +... diff --git a/prometheus-mysql-exporter/.helmignore b/prometheus-mysql-exporter/.helmignore new file mode 100644 index 0000000000..f0c1319444 --- /dev/null +++ b/prometheus-mysql-exporter/.helmignore @@ -0,0 +1,21 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj diff --git a/prometheus-mysql-exporter/Chart.yaml b/prometheus-mysql-exporter/Chart.yaml new file mode 100644 index 0000000000..6a055f1840 --- /dev/null +++ b/prometheus-mysql-exporter/Chart.yaml @@ -0,0 +1,30 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: v0.12.1 +description: OpenStack-Helm Prometheus mysql-exporter +name: prometheus-mysql-exporter +version: 2024.2.0 +home: https://mariadb.com/kb/en/ +icon: http://badges.mariadb.org/mariadb-badge-180x60.png +sources: + - https://github.com/MariaDB/server + - https://opendev.org/openstack/openstack-helm +maintainers: + - name: OpenStack-Helm Authors +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit + version: ">= 0.1.0" +... diff --git a/prometheus-mysql-exporter/README.rst b/prometheus-mysql-exporter/README.rst new file mode 100644 index 0000000000..1615a9065c --- /dev/null +++ b/prometheus-mysql-exporter/README.rst @@ -0,0 +1,18 @@ +openstack-helm/mariadb +====================== + +By default, this chart creates a 3-member mariadb galera cluster. + +This chart depends on mariadb-operator chart. + +The StatefulSets all leverage PVCs to provide stateful storage to +``/var/lib/mysql``. + +You must ensure that your control nodes that should receive mariadb +instances are labeled with ``openstack-control-plane=enabled``, or +whatever you have configured in values.yaml for the label +configuration: + +:: + + kubectl label nodes openstack-control-plane=enabled --all diff --git a/prometheus-mysql-exporter/templates/bin/_create-mysql-user.sh.tpl b/prometheus-mysql-exporter/templates/bin/_create-mysql-user.sh.tpl new file mode 100644 index 0000000000..bf6e733cbc --- /dev/null +++ b/prometheus-mysql-exporter/templates/bin/_create-mysql-user.sh.tpl @@ -0,0 +1,50 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -e + + # SLAVE MONITOR + # Grants ability to SHOW SLAVE STATUS, SHOW REPLICA STATUS, + # SHOW ALL SLAVES STATUS, SHOW ALL REPLICAS STATUS, SHOW RELAYLOG EVENTS. + # New privilege added in MariaDB Enterprise Server 10.5.8-5. Alias for REPLICA MONITOR. + # + # REPLICATION CLIENT + # Grants ability to SHOW MASTER STATUS, SHOW SLAVE STATUS, SHOW BINARY LOGS. In ES10.5, + # is an alias for BINLOG MONITOR and the capabilities have changed. BINLOG MONITOR grants + # ability to SHOW MASTER STATUS, SHOW BINARY LOGS, SHOW BINLOG EVENTS, and SHOW BINLOG STATUS. + + mariadb_version=$(mysql --defaults-file=/etc/mysql/admin_user.cnf -e "status" | grep -E '^Server\s+version:') + echo "Current database ${mariadb_version}" + + if [[ ! -z ${mariadb_version} && -z $(grep -E '10.2|10.3|10.4' <<< ${mariadb_version}) ]]; then + # In case MariaDB version is 10.2.x-10.4.x - we use old privileges definitions + if ! mysql --defaults-file=/etc/mysql/admin_user.cnf -e \ + "CREATE OR REPLACE USER '${EXPORTER_USER}'@'%' IDENTIFIED BY '${EXPORTER_PASSWORD}'; \ + GRANT PROCESS, BINLOG MONITOR, SLAVE MONITOR, SELECT ON *.* TO '${EXPORTER_USER}'@'%' ${MARIADB_X509}; \ + FLUSH PRIVILEGES;" ; then + echo "ERROR: Could not create user: ${EXPORTER_USER}" + exit 1 + fi + else + # here we use new MariaDB privileges definitions defines since version 10.5 + if ! mysql --defaults-file=/etc/mysql/admin_user.cnf -e \ + "CREATE OR REPLACE USER '${EXPORTER_USER}'@'%' IDENTIFIED BY '${EXPORTER_PASSWORD}'; \ + GRANT PROCESS, REPLICATION CLIENT, SELECT ON *.* TO '${EXPORTER_USER}'@'%' ${MARIADB_X509}; \ + FLUSH PRIVILEGES;" ; then + echo "ERROR: Could not create user: ${EXPORTER_USER}" + exit 1 + fi + fi diff --git a/prometheus-mysql-exporter/templates/bin/_mysqld-exporter.sh.tpl b/prometheus-mysql-exporter/templates/bin/_mysqld-exporter.sh.tpl new file mode 100644 index 0000000000..d794be3749 --- /dev/null +++ b/prometheus-mysql-exporter/templates/bin/_mysqld-exporter.sh.tpl @@ -0,0 +1,57 @@ +#!/bin/sh + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +compareVersions() { +echo $1 $2 | \ +awk '{ split($1, a, "."); + split($2, b, "."); + res = -1; + for (i = 1; i <= 3; i++){ + if (a[i] < b[i]) { + res =-1; + break; + } else if (a[i] > b[i]) { + res = 1; + break; + } else if (a[i] == b[i]) { + if (i == 3) { + res = 0; + break; + } else { + continue; + } + } + } + print res; + }' +} + +MYSQL_EXPORTER_VER=`/bin/mysqld_exporter --version 2>&1 | grep "mysqld_exporter" | awk '{print $3}'` + +#in versions greater than 0.10.0 different configuration flags are used: +#https://github.com/prometheus/mysqld_exporter/commit/66c41ac7eb90a74518a6ecf6c6bb06464eb68db8 +compverResult=`compareVersions "${MYSQL_EXPORTER_VER}" "0.10.0"` +CONFIG_FLAG_PREFIX='-' +if [ ${compverResult} -gt 0 ]; then + CONFIG_FLAG_PREFIX='--' +fi + +exec /bin/mysqld_exporter \ + ${CONFIG_FLAG_PREFIX}config.my-cnf=/etc/mysql/mysql_user.cnf \ + ${CONFIG_FLAG_PREFIX}web.listen-address="${POD_IP}:${LISTEN_PORT}" \ + ${CONFIG_FLAG_PREFIX}web.telemetry-path="$TELEMETRY_PATH" diff --git a/prometheus-mysql-exporter/templates/exporter-configmap-bin.yaml b/prometheus-mysql-exporter/templates/exporter-configmap-bin.yaml new file mode 100644 index 0000000000..94bafc0ba0 --- /dev/null +++ b/prometheus-mysql-exporter/templates/exporter-configmap-bin.yaml @@ -0,0 +1,27 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.monitoring.prometheus.configmap_bin .Values.monitoring.prometheus.enabled }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: mysql-exporter-bin +data: + create-mysql-user.sh: | +{{ tuple "bin/_create-mysql-user.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + mysqld-exporter.sh: | +{{ tuple "bin/_mysqld-exporter.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} diff --git a/prometheus-mysql-exporter/templates/exporter-deployment.yaml b/prometheus-mysql-exporter/templates/exporter-deployment.yaml new file mode 100644 index 0000000000..b2ac8242f5 --- /dev/null +++ b/prometheus-mysql-exporter/templates/exporter-deployment.yaml @@ -0,0 +1,103 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.monitoring.prometheus.deployment_exporter .Values.monitoring.prometheus.enabled }} +{{- $envAll := . }} + +{{- $serviceAccountName := "prometheus-mysql-exporter" }} +{{ tuple $envAll "prometheus_mysql_exporter" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus-mysql-exporter + labels: +{{ tuple $envAll "prometheus-mysql-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + replicas: {{ .Values.pod.replicas.prometheus_mysql_exporter }} + selector: + matchLabels: +{{ tuple $envAll "prometheus-mysql-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} +{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "prometheus-mysql-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + namespace: {{ .Values.endpoints.prometheus_mysql_exporter.namespace }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} +{{ dict "envAll" $envAll "podName" "prometheus-mysql-exporter" "containerNames" (list "init" "mysql-exporter") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: + shareProcessNamespace: true + serviceAccountName: {{ $serviceAccountName }} +{{ dict "envAll" $envAll "application" "prometheus_mysql_exporter" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + nodeSelector: + {{ .Values.labels.prometheus_mysql_exporter.node_selector_key }}: {{ .Values.labels.prometheus_mysql_exporter.node_selector_value }} + terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.prometheus_mysql_exporter.timeout | default "30" }} + initContainers: +{{ tuple $envAll "prometheus_mysql_exporter" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: mysql-exporter +{{ tuple $envAll "prometheus_mysql_exporter" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ dict "envAll" $envAll "application" "prometheus_mysql_exporter" "container" "exporter" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.prometheus_mysql_exporter | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + command: + - /tmp/mysqld-exporter.sh + ports: + - name: metrics + containerPort: {{ tuple "prometheus_mysql_exporter" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + env: + - name: EXPORTER_USER + valueFrom: + secretKeyRef: + name: mysql-exporter-secrets + key: EXPORTER_USER + - name: EXPORTER_PASSWORD + valueFrom: + secretKeyRef: + name: mysql-exporter-secrets + key: EXPORTER_PASSWORD + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: LISTEN_PORT + value: {{ tuple "prometheus_mysql_exporter" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: TELEMETRY_PATH + value: {{ tuple "prometheus_mysql_exporter" "internal" "metrics" . | include "helm-toolkit.endpoints.keystone_endpoint_path_lookup" | quote }} + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: mysql-exporter-secrets + mountPath: /etc/mysql/mysql_user.cnf + subPath: mysql_user.cnf + readOnly: true + - name: mysql-exporter-bin + mountPath: /tmp/mysqld-exporter.sh + subPath: mysqld-exporter.sh + readOnly: true +{{ dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.secrets.tls.oslo_db.server.internal "path" "/etc/mysql/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: mysql-exporter-secrets + secret: + secretName: mysql-exporter-secrets + defaultMode: 0444 + - name: mysql-exporter-bin + configMap: + name: mysql-exporter-bin + defaultMode: 0555 +{{ dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.secrets.tls.oslo_db.server.internal | include "helm-toolkit.snippets.tls_volume" | indent 8 }} +{{- end }} diff --git a/prometheus-mysql-exporter/templates/exporter-job-create-user.yaml b/prometheus-mysql-exporter/templates/exporter-job-create-user.yaml new file mode 100644 index 0000000000..3352ab8d6a --- /dev/null +++ b/prometheus-mysql-exporter/templates/exporter-job-create-user.yaml @@ -0,0 +1,98 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.monitoring.prometheus.job_user_create .Values.monitoring.prometheus.enabled }} +{{- $envAll := . }} + +{{- $serviceAccountName := "exporter-create-sql-user" }} +{{ tuple $envAll "prometheus_create_mysql_user" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: exporter-create-sql-user + labels: +{{ tuple $envAll "prometheus-mysql-exporter" "create-sql-user" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- if .Values.helm3_hook }} + annotations: + "helm.sh/hook": "post-install,post-upgrade" + "helm.sh/hook-weight": "5" + "helm.sh/hook-delete-policy": "before-hook-creation" +{{- end }} +spec: + backoffLimit: {{ .Values.jobs.exporter_create_sql_user.backoffLimit }} + template: + metadata: + labels: +{{ tuple $envAll "prometheus-mysql-exporter" "create-sql-user" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} +{{ dict "envAll" $envAll "podName" "create-sql-user" "containerNames" (list "init" "exporter-create-sql-user") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: + shareProcessNamespace: true + serviceAccountName: {{ $serviceAccountName }} +{{ dict "envAll" $envAll "application" "prometheus_create_mysql_user" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + activeDeadlineSeconds: {{ .Values.jobs.exporter_create_sql_user.activeDeadlineSeconds }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.prometheus_mysql_exporter.node_selector_key }}: {{ .Values.labels.prometheus_mysql_exporter.node_selector_value }} + initContainers: +{{ tuple $envAll "prometheus_create_mysql_user" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: exporter-create-sql-user +{{ tuple $envAll "prometheus_create_mysql_user" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ dict "envAll" $envAll "application" "prometheus_create_mysql_user" "container" "main" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.prometheus_create_mysql_user | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + command: + - /tmp/create-mysql-user.sh + env: + - name: EXPORTER_USER + valueFrom: + secretKeyRef: + name: mysql-exporter-secrets + key: EXPORTER_USER + - name: EXPORTER_PASSWORD + valueFrom: + secretKeyRef: + name: mysql-exporter-secrets + key: EXPORTER_PASSWORD +{{- if $envAll.Values.manifests.certificates }} + - name: MARIADB_X509 + value: "REQUIRE X509" +{{- end }} + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: mysql-exporter-bin + mountPath: /tmp/create-mysql-user.sh + subPath: create-mysql-user.sh + readOnly: true + - name: mariadb-secrets + mountPath: /etc/mysql/admin_user.cnf + subPath: admin_user.cnf + readOnly: true +{{ dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.secrets.tls.oslo_db.server.internal "path" "/etc/mysql/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: mysql-exporter-bin + configMap: + name: mysql-exporter-bin + defaultMode: 0555 + - name: mariadb-secrets + secret: + secretName: mariadb-secrets + defaultMode: 0444 +{{ dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.secrets.tls.oslo_db.server.internal | include "helm-toolkit.snippets.tls_volume" | indent 8 }} +{{- end }} diff --git a/prometheus-mysql-exporter/templates/exporter-network-policy.yaml b/prometheus-mysql-exporter/templates/exporter-network-policy.yaml new file mode 100644 index 0000000000..3769506e70 --- /dev/null +++ b/prometheus-mysql-exporter/templates/exporter-network-policy.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.monitoring.prometheus.network_policy_exporter .Values.monitoring.prometheus.enabled -}} +{{- $netpol_opts := dict "envAll" . "name" "application" "label" "prometheus-mysql-exporter" -}} +{{ $netpol_opts | include "helm-toolkit.manifests.kubernetes_network_policy" }} +{{- end -}} diff --git a/prometheus-mysql-exporter/templates/exporter-secrets-etc.yaml b/prometheus-mysql-exporter/templates/exporter-secrets-etc.yaml new file mode 100644 index 0000000000..99f01f8e2c --- /dev/null +++ b/prometheus-mysql-exporter/templates/exporter-secrets-etc.yaml @@ -0,0 +1,33 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.monitoring.prometheus.secret_etc .Values.monitoring.prometheus.enabled }} +{{- $envAll := . }} + +{{- $exporter_user := .Values.endpoints.oslo_db.auth.exporter.username }} +{{- $exporter_password := .Values.endpoints.oslo_db.auth.exporter.password }} +{{- $db_host := tuple "oslo_db" "direct" "mysql" $envAll | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }} +{{- $data_source_name := printf "%s:%s@(%s)/" $exporter_user $exporter_password $db_host }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: mysql-exporter-secrets +type: Opaque +data: + DATA_SOURCE_NAME: {{ $data_source_name | b64enc }} + EXPORTER_USER: {{ .Values.endpoints.oslo_db.auth.exporter.username | b64enc }} + EXPORTER_PASSWORD: {{ .Values.endpoints.oslo_db.auth.exporter.password | b64enc }} + mysql_user.cnf: {{ tuple "secrets/_exporter_user.cnf.tpl" . | include "helm-toolkit.utils.template" | b64enc }} +{{- end }} diff --git a/prometheus-mysql-exporter/templates/exporter-service.yaml b/prometheus-mysql-exporter/templates/exporter-service.yaml new file mode 100644 index 0000000000..a7166358ad --- /dev/null +++ b/prometheus-mysql-exporter/templates/exporter-service.yaml @@ -0,0 +1,35 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.monitoring.prometheus.service_exporter .Values.monitoring.prometheus.enabled }} +{{- $envAll := . }} +{{- $prometheus_annotations := $envAll.Values.monitoring.prometheus.mysqld_exporter }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "prometheus_mysql_exporter" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + labels: +{{ tuple $envAll "prometheus-mysql-exporter" "metrics" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: +{{- if .Values.monitoring.prometheus.enabled }} +{{ tuple $prometheus_annotations | include "helm-toolkit.snippets.prometheus_service_annotations" | indent 4 }} +{{- end }} +spec: + ports: + - name: metrics + port: {{ tuple "prometheus_mysql_exporter" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + selector: +{{ tuple $envAll "prometheus-mysql-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- end }} diff --git a/prometheus-mysql-exporter/templates/secrets/_exporter_user.cnf.tpl b/prometheus-mysql-exporter/templates/secrets/_exporter_user.cnf.tpl new file mode 100644 index 0000000000..c86fc01f25 --- /dev/null +++ b/prometheus-mysql-exporter/templates/secrets/_exporter_user.cnf.tpl @@ -0,0 +1,24 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +[client] +user = {{ .Values.endpoints.oslo_db.auth.exporter.username }} +password = {{ .Values.endpoints.oslo_db.auth.exporter.password }} +host = {{ tuple "oslo_db" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }} +port = {{ tuple "oslo_db" "direct" "mysql" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +{{- if .Values.manifests.certificates }} +ssl-ca = /etc/mysql/certs/ca.crt +ssl-key = /etc/mysql/certs/tls.key +ssl-cert = /etc/mysql/certs/tls.crt +{{- end }} diff --git a/prometheus-mysql-exporter/values.yaml b/prometheus-mysql-exporter/values.yaml new file mode 100644 index 0000000000..431e9dcca4 --- /dev/null +++ b/prometheus-mysql-exporter/values.yaml @@ -0,0 +1,329 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for mariadb. +# This is a YAML-formatted file. +# Declare name/value pairs to be passed into your templates. +# name: value + +--- +release_group: null + +images: + tags: + prometheus_create_mysql_user: docker.io/library/mariadb:10.5.9-focal + prometheus_mysql_exporter: docker.io/prom/mysqld-exporter:v0.12.1 + prometheus_mysql_exporter_helm_tests: docker.io/openstackhelm/heat:wallaby-ubuntu_focal + dep_check: quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal + image_repo_sync: docker.io/library/docker:17.07.0 + pull_policy: "IfNotPresent" + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +labels: + prometheus_mysql_exporter: + node_selector_key: openstack-control-plane + node_selector_value: enabled + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +pod: + security_context: + prometheus_mysql_exporter: + pod: + runAsUser: 99 + container: + exporter: + runAsUser: 99 + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + prometheus_create_mysql_user: + pod: + runAsUser: 0 + container: + main: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + weight: + default: 10 + replicas: + prometheus_mysql_exporter: 1 + lifecycle: + upgrades: + deployments: + revision_history: 3 + pod_replacement_strategy: RollingUpdate + rolling_update: + max_unavailable: 1 + max_surge: 3 + termination_grace_period: + prometheus_mysql_exporter: + timeout: 30 + resources: + enabled: false + prometheus_mysql_exporter: + limits: + memory: "1024Mi" + cpu: "2000m" + requests: + memory: "128Mi" + cpu: "500m" + jobs: + prometheus_create_mysql_user: + limits: + memory: "1024Mi" + cpu: "2000m" + requests: + memory: "128Mi" + cpu: "100m" + image_repo_sync: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - mysql-exporter-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + prometheus_create_mysql_user: + services: + - endpoint: internal + service: oslo_db + prometheus_mysql_exporter: + jobs: + - exporter-create-sql-user + services: + - endpoint: internal + service: oslo_db + prometheus_mysql_exporter_tests: + services: + - endpoint: internal + service: prometheus_mysql_exporter + - endpoint: internal + service: monitoring + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + +jobs: + exporter_create_sql_user: + backoffLimit: 87600 + activeDeadlineSeconds: 3600 + +monitoring: + prometheus: + enabled: false + mysqld_exporter: + scrape: true + +secrets: + identity: + admin: keystone-admin-user + oci_image_registry: + mariadb: mariadb-oci-image-registry-key + tls: + oslo_db: + server: + public: mariadb-tls-server + internal: mariadb-tls-direct + +# typically overridden by environmental +# values, but should include all endpoints +# required by this chart +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + mariadb: + username: mariadb + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + monitoring: + name: prometheus + namespace: null + hosts: + default: prom-metrics + public: prometheus + host_fqdn_override: + default: null + path: + default: null + scheme: + default: 'http' + port: + api: + default: 9090 + public: 80 + prometheus_mysql_exporter: + namespace: null + hosts: + default: mysql-exporter + host_fqdn_override: + default: null + path: + default: /metrics + scheme: + default: 'http' + port: + metrics: + default: 9104 + oslo_db: + namespace: null + auth: + admin: + username: root + password: password + sst: + username: sst + password: password + audit: + username: audit + password: password + exporter: + username: exporter + password: password + hosts: + default: mariadb-server-primary + direct: mariadb-server-internal + discovery: mariadb-discovery + server: mariadb-server + host_fqdn_override: + default: null + path: null + scheme: mysql+pymysql + port: + mysql: + default: 3306 + wsrep: + default: 4567 + kube_dns: + namespace: kube-system + name: kubernetes-dns + hosts: + default: kube-dns + host_fqdn_override: + default: null + path: + default: null + scheme: http + port: + dns_tcp: + default: 53 + dns: + default: 53 + protocol: UDP + identity: + name: backup-storage-auth + namespace: openstack + auth: + admin: + # Auth URL of null indicates local authentication + # HTK will form the URL unless specified here + auth_url: null + region_name: RegionOne + username: admin + password: password + project_name: admin + user_domain_name: default + project_domain_name: default + mariadb-server: + # Auth URL of null indicates local authentication + # HTK will form the URL unless specified here + auth_url: null + role: admin + region_name: RegionOne + username: mariadb-backup-user + password: password + project_name: service + user_domain_name: service + project_domain_name: service + hosts: + default: keystone + internal: keystone-api + host_fqdn_override: + default: null + path: + default: /v3 + scheme: + default: 'http' + port: + api: + default: 80 + internal: 5000 + +network_policy: + prometheus-mysql-exporter: + ingress: + - {} + egress: + - {} + +# Helm hook breaks for helm2. +# Set helm3_hook: false in case helm2 is used. +helm3_hook: true + +manifests: + certificates: false + job_image_repo_sync: true + monitoring: + prometheus: + configmap_bin: false + deployment_exporter: false + job_user_create: false + secret_etc: false + service_exporter: false + network_policy_exporter: false + network_policy: false + secret_etc: true + secret_registry: true +... diff --git a/prometheus-node-exporter/Chart.yaml b/prometheus-node-exporter/Chart.yaml new file mode 100644 index 0000000000..3030497cc2 --- /dev/null +++ b/prometheus-node-exporter/Chart.yaml @@ -0,0 +1,29 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: v0.18.1 +description: OpenStack-Helm Node Exporter for Prometheus +name: prometheus-node-exporter +version: 2024.2.0 +home: https://github.com/prometheus/node_exporter +sources: + - https://github.com/prometheus/node_exporter + - https://opendev.org/openstack/openstack-helm-infra +maintainers: + - name: OpenStack-Helm Authors +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit + version: ">= 0.1.0" +... diff --git a/prometheus-node-exporter/templates/bin/_node-exporter.sh.tpl b/prometheus-node-exporter/templates/bin/_node-exporter.sh.tpl new file mode 100644 index 0000000000..7b268d690a --- /dev/null +++ b/prometheus-node-exporter/templates/bin/_node-exporter.sh.tpl @@ -0,0 +1,34 @@ +#!/bin/sh +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +exec /bin/node_exporter \ + {{- if .Values.conf.collectors.enable }} + {{ tuple "--collector." .Values.conf.collectors.enable | include "helm-toolkit.utils.joinListWithPrefix" }} \ + {{- end }} + {{- if .Values.conf.collectors.disable }} + {{ tuple "--no-collector." .Values.conf.collectors.disable | include "helm-toolkit.utils.joinListWithPrefix" }} \ + {{- end }} + {{- if .Values.conf.collectors.textfile.directory }} + --collector.textfile.directory={{.Values.conf.collectors.textfile.directory }} \ + {{- end }} + {{- if .Values.conf.collectors.filesystem.ignored_mount_points }} + --collector.filesystem.ignored-mount-points={{ .Values.conf.collectors.filesystem.ignored_mount_points }} \ + {{- end }} + {{- if .Values.conf.collectors.filesystem.rootfs_mount_point }} + --path.rootfs={{ .Values.conf.collectors.filesystem.rootfs_mount_point }} \ + {{- end }} + --collector.ntp.server={{ .Values.conf.ntp_server_ip }} diff --git a/prometheus-node-exporter/templates/configmap-bin.yaml b/prometheus-node-exporter/templates/configmap-bin.yaml new file mode 100644 index 0000000000..f31a2ab0b4 --- /dev/null +++ b/prometheus-node-exporter/templates/configmap-bin.yaml @@ -0,0 +1,27 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_bin }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: node-exporter-bin +data: + node-exporter.sh: | +{{ tuple "bin/_node-exporter.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + image-repo-sync.sh: | +{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} +{{- end }} diff --git a/prometheus-node-exporter/templates/daemonset.yaml b/prometheus-node-exporter/templates/daemonset.yaml new file mode 100644 index 0000000000..e37cf892ce --- /dev/null +++ b/prometheus-node-exporter/templates/daemonset.yaml @@ -0,0 +1,124 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.daemonset }} +{{- $envAll := . }} + +{{- $mounts_node_exporter := .Values.pod.mounts.node_exporter.node_exporter}} + +{{- $serviceAccountName := printf "%s-%s" .Release.Name "node-exporter" }} +{{ tuple $envAll "node_exporter" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: run-node-exporter +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: cluster-admin + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: node-exporter + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "node_exporter" "metrics" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + selector: + matchLabels: +{{ tuple $envAll "node_exporter" "metrics" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} +{{ tuple $envAll "node_exporter" | include "helm-toolkit.snippets.kubernetes_upgrades_daemonset" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "node_exporter" "metrics" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ dict "envAll" $envAll "podName" "node-exporter" "containerNames" (list "node-exporter" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + spec: +{{ dict "envAll" $envAll "application" "metrics" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} +{{ if .Values.pod.tolerations.node_exporter.enabled }} +{{ tuple $envAll "node_exporter" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }} +{{ else }} + nodeSelector: + {{ .Values.labels.node_exporter.node_selector_key }}: {{ .Values.labels.node_exporter.node_selector_value | quote }} +{{ end }} + hostNetwork: true + hostPID: true + initContainers: +{{ tuple $envAll "node_exporter" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: node-exporter +{{ tuple $envAll "node_exporter" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.node_exporter | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "metrics" "container" "node_exporter" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/node-exporter.sh + ports: + - name: metrics + containerPort: {{ tuple "node_metrics" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + hostPort: {{ tuple "node_metrics" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + readinessProbe: + httpGet: + port: {{ tuple "node_metrics" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + initialDelaySeconds: 20 + periodSeconds: 10 + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: proc + mountPath: /host/proc + readOnly: true + - name: sys + mountPath: /host/sys + readOnly: true +{{ if .Values.conf.collectors.textfile.directory }} + - name: stats-out + mountPath: {{.Values.conf.collectors.textfile.directory }} + readOnly: true +{{ end }} + - name: node-exporter-bin + mountPath: /tmp/node-exporter.sh + subPath: node-exporter.sh + readOnly: true +{{ if $mounts_node_exporter.volumeMounts }}{{ toYaml $mounts_node_exporter.volumeMounts | indent 12 }}{{ end }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: proc + hostPath: + path: /proc + - name: sys + hostPath: + path: /sys +{{ if .Values.conf.collectors.textfile.directory }} + - name: stats-out + hostPath: + path: {{.Values.conf.collectors.textfile.directory }} +{{ end }} + - name: node-exporter-bin + configMap: + name: node-exporter-bin + defaultMode: 0555 +{{ if $mounts_node_exporter.volumes }}{{ toYaml $mounts_node_exporter.volumes | indent 8 }}{{ end }} +{{- end }} diff --git a/prometheus-node-exporter/templates/job-image-repo-sync.yaml b/prometheus-node-exporter/templates/job-image-repo-sync.yaml new file mode 100644 index 0000000000..a9ca9f5d07 --- /dev/null +++ b/prometheus-node-exporter/templates/job-image-repo-sync.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "node-exporter" -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/prometheus-node-exporter/templates/secret-registry.yaml b/prometheus-node-exporter/templates/secret-registry.yaml new file mode 100644 index 0000000000..da979b3223 --- /dev/null +++ b/prometheus-node-exporter/templates/secret-registry.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_registry .Values.endpoints.oci_image_registry.auth.enabled }} +{{ include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) }} +{{- end }} diff --git a/prometheus-node-exporter/templates/service.yaml b/prometheus-node-exporter/templates/service.yaml new file mode 100644 index 0000000000..f615d576ce --- /dev/null +++ b/prometheus-node-exporter/templates/service.yaml @@ -0,0 +1,38 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.service }} +{{- $envAll := . }} +{{- $prometheus_annotations := $envAll.Values.monitoring.prometheus.node_exporter }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "node_metrics" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + labels: +{{ tuple $envAll "node_exporter" "metrics" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: +{{- if .Values.monitoring.prometheus.enabled }} +{{ tuple $prometheus_annotations | include "helm-toolkit.snippets.prometheus_service_annotations" | indent 4 }} +{{- end }} +spec: + type: ClusterIP + clusterIP: None + ports: + - name: metrics + port: {{ tuple "node_metrics" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + targetPort: {{ tuple "node_metrics" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + selector: +{{ tuple $envAll "node_exporter" "metrics" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- end }} diff --git a/prometheus-node-exporter/values.yaml b/prometheus-node-exporter/values.yaml new file mode 100644 index 0000000000..bcba31909b --- /dev/null +++ b/prometheus-node-exporter/values.yaml @@ -0,0 +1,186 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for node-exporter. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +--- +images: + tags: + node_exporter: docker.io/prom/node-exporter:v0.18.1 + dep_check: quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal + image_repo_sync: docker.io/library/docker:17.07.0 + pull_policy: IfNotPresent + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +labels: + node_exporter: + node_selector_key: openstack-control-plane + node_selector_value: enabled + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +pod: + security_context: + metrics: + pod: + runAsUser: 65534 + container: + node_exporter: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + mounts: + node_exporter: + node_exporter: + init_container: null + lifecycle: + upgrades: + daemonsets: + pod_replacement_strategy: RollingUpdate + node_exporter: + enabled: true + min_ready_seconds: 0 + max_unavailable: 1 + termination_grace_period: + node_exporter: + timeout: 30 + resources: + enabled: false + node_exporter: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + jobs: + image_repo_sync: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + tolerations: + node_exporter: + enabled: false + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + - key: node-role.kubernetes.io/control-plane + operator: Exists + - key: node-role.kubernetes.io/node + operator: Exists +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - node-exporter-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + node_exporter: + services: null + +monitoring: + prometheus: + enabled: true + node_exporter: + scrape: true + +secrets: + oci_image_registry: + prometheus-node-exporter: prometheus-node-exporter-oci-image-registry-key + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + prometheus-node-exporter: + username: prometheus-node-exporter + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + node_metrics: + namespace: null + hosts: + default: node-exporter + host_fqdn_override: + default: null + path: + default: null + scheme: + default: 'http' + port: + metrics: + default: 9100 + +manifests: + configmap_bin: true + daemonset: true + job_image_repo_sync: true + secret_registry: true + service: true + +conf: + ntp_server_ip: 127.0.0.1 + collectors: + enable: + - ntp + - meminfo_numa + - bonding + - mountstats + disable: + textfile: + directory: /var/log/node-exporter-vfstats + filesystem: + ignored_mount_points: + rootfs_mount_point: +... diff --git a/prometheus-openstack-exporter/Chart.yaml b/prometheus-openstack-exporter/Chart.yaml new file mode 100644 index 0000000000..93a1825f42 --- /dev/null +++ b/prometheus-openstack-exporter/Chart.yaml @@ -0,0 +1,29 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: v1.0.0 +description: OpenStack Metrics Exporter for Prometheus +name: prometheus-openstack-exporter +version: 2024.2.0 +home: https://github.com/openstack/openstack-helm-infra +sources: + - https://opendev.org/openstack/openstack-helm-infra + - https://github.com/rakesh-patnaik/prometheus-openstack-exporter +maintainers: + - name: OpenStack-Helm Authors +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit + version: ">= 0.1.0" +... diff --git a/prometheus-openstack-exporter/templates/bin/_prometheus-openstack-exporter.sh.tpl b/prometheus-openstack-exporter/templates/bin/_prometheus-openstack-exporter.sh.tpl new file mode 100644 index 0000000000..0868403fe9 --- /dev/null +++ b/prometheus-openstack-exporter/templates/bin/_prometheus-openstack-exporter.sh.tpl @@ -0,0 +1,28 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex +COMMAND="${@:-start}" + +function start () { + exec python3 /usr/local/bin/exporter/main.py +} + +function stop () { + kill -TERM 1 +} + +$COMMAND diff --git a/prometheus-openstack-exporter/templates/configmap-bin.yaml b/prometheus-openstack-exporter/templates/configmap-bin.yaml new file mode 100644 index 0000000000..e833f93352 --- /dev/null +++ b/prometheus-openstack-exporter/templates/configmap-bin.yaml @@ -0,0 +1,29 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_bin }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-openstack-exporter-bin +data: + image-repo-sync.sh: | +{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} + ks-user.sh: | +{{- include "helm-toolkit.scripts.keystone_user" . | indent 4 }} + prometheus-openstack-exporter.sh: | +{{ tuple "bin/_prometheus-openstack-exporter.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} diff --git a/prometheus-openstack-exporter/templates/deployment.yaml b/prometheus-openstack-exporter/templates/deployment.yaml new file mode 100644 index 0000000000..a9b0391f1f --- /dev/null +++ b/prometheus-openstack-exporter/templates/deployment.yaml @@ -0,0 +1,108 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.deployment }} +{{- $envAll := . }} +{{- $ksUserSecret := .Values.secrets.identity.user }} + +{{- $serviceAccountName := "prometheus-openstack-exporter" }} +{{ tuple $envAll "prometheus_openstack_exporter" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus-openstack-exporter + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "prometheus-openstack-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + replicas: {{ .Values.pod.replicas.prometheus_openstack_exporter }} + selector: + matchLabels: +{{ tuple $envAll "prometheus-openstack-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} +{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "prometheus-openstack-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + secret-keystone-hash: {{ tuple "secret-keystone.yaml" . | include "helm-toolkit.utils.hash" }} + secret-registry-hash: {{ tuple "secret-registry.yaml" . | include "helm-toolkit.utils.hash" }} +{{ dict "envAll" $envAll "podName" "prometheus-openstack-exporter" "containerNames" (list "openstack-metrics-exporter" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "exporter" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + affinity: +{{ tuple $envAll "prometheus-openstack-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} + nodeSelector: + {{ .Values.labels.openstack_exporter.node_selector_key }}: {{ .Values.labels.openstack_exporter.node_selector_value | quote }} + terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.prometheus_openstack_exporter.timeout | default "30" }} + initContainers: +{{ tuple $envAll "prometheus_openstack_exporter" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: openstack-metrics-exporter +{{ tuple $envAll "prometheus_openstack_exporter" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.prometheus_openstack_exporter | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "exporter" "container" "openstack_metrics_exporter" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/prometheus-openstack-exporter.sh + - start + lifecycle: + preStop: + exec: + command: + - /tmp/prometheus-openstack-exporter.sh + - stop + ports: + - name: metrics + containerPort: {{ tuple "prometheus_openstack_exporter" "internal" "exporter" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + livenessProbe: + httpGet: + path: /metrics + port: {{ tuple "prometheus_openstack_exporter" "internal" "exporter" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + initialDelaySeconds: 180 + periodSeconds: 60 + readinessProbe: + httpGet: + path: /metrics + port: {{ tuple "prometheus_openstack_exporter" "internal" "exporter" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + initialDelaySeconds: 20 + periodSeconds: 30 + env: + - name: LISTEN_PORT + value: {{ tuple "prometheus_openstack_exporter" "internal" "exporter" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} +{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.conf.prometheus_openstack_exporter | indent 12 }} +{{- with $env := dict "ksUserSecret" $ksUserSecret "useCA" .Values.manifests.certificates }} +{{- include "helm-toolkit.snippets.keystone_openrc_env_vars" $env | indent 12 }} +{{- end }} + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: prometheus-openstack-exporter-bin + mountPath: /tmp/prometheus-openstack-exporter.sh + subPath: prometheus-openstack-exporter.sh + readOnly: true +{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.identity.api.internal | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: prometheus-openstack-exporter-bin + configMap: + name: prometheus-openstack-exporter-bin + defaultMode: 0555 +{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.identity.api.internal | include "helm-toolkit.snippets.tls_volume" | indent 8 }} +{{- end }} diff --git a/prometheus-openstack-exporter/templates/job-image-repo-sync.yaml b/prometheus-openstack-exporter/templates/job-image-repo-sync.yaml new file mode 100644 index 0000000000..4c77ef2160 --- /dev/null +++ b/prometheus-openstack-exporter/templates/job-image-repo-sync.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "prometheus-openstack-exporter" -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/prometheus-openstack-exporter/templates/job-ks-user.yaml b/prometheus-openstack-exporter/templates/job-ks-user.yaml new file mode 100644 index 0000000000..04d126a3f5 --- /dev/null +++ b/prometheus-openstack-exporter/templates/job-ks-user.yaml @@ -0,0 +1,76 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.job_ks_user }} +{{- $envAll := . }} + +{{- $serviceAccountName := "prometheus-openstack-exporter-ks-user" }} +{{ tuple $envAll "ks_user" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: prometheus-openstack-exporter-ks-user + labels: +{{ tuple $envAll "prometheus-openstack-exporter" "ks-user" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + template: + metadata: + labels: +{{ tuple $envAll "prometheus-openstack-exporter" "ks-user" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ dict "envAll" $envAll "podName" "prometheus-openstack-exporter-ks-user" "containerNames" (list "prometheus-openstack-exporter-ks-user" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "ks_user" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value | quote }} + initContainers: +{{ tuple $envAll "ks_user" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: prometheus-openstack-exporter-ks-user +{{ tuple $envAll "ks_user" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ dict "envAll" $envAll "application" "ks_user" "container" "prometheus_openstack_exporter_ks_user" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/ks-user.sh +{{ tuple $envAll $envAll.Values.pod.resources.jobs.ks_user | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: ks-user-sh + mountPath: /tmp/ks-user.sh + subPath: ks-user.sh + readOnly: true +{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.identity.api.internal | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }} + env: +{{- with $env := dict "ksUserSecret" .Values.secrets.identity.admin "useCA" .Values.manifests.certificates }} +{{- include "helm-toolkit.snippets.keystone_openrc_env_vars" $env | indent 12 }} +{{- end }} + - name: SERVICE_OS_SERVICE_NAME + value: "prometheus-openstack-exporter" +{{- with $env := dict "ksUserSecret" .Values.secrets.identity.user }} +{{- include "helm-toolkit.snippets.keystone_user_create_env_vars" $env | indent 12 }} +{{- end }} + - name: SERVICE_OS_ROLE + value: {{ .Values.endpoints.identity.auth.user.role | quote }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: ks-user-sh + configMap: + name: prometheus-openstack-exporter-bin + defaultMode: 0555 +{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.identity.api.internal | include "helm-toolkit.snippets.tls_volume" | indent 8 }} +{{- end }} diff --git a/prometheus-openstack-exporter/templates/network_policy.yaml b/prometheus-openstack-exporter/templates/network_policy.yaml new file mode 100644 index 0000000000..9e19a196d6 --- /dev/null +++ b/prometheus-openstack-exporter/templates/network_policy.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */}} + +{{- if .Values.manifests.network_policy -}} +{{- $netpol_opts := dict "envAll" . "name" "application" "label" "prometheus-openstack-exporter" -}} +{{ $netpol_opts | include "helm-toolkit.manifests.kubernetes_network_policy" }} +{{- end -}} diff --git a/prometheus-openstack-exporter/templates/secret-keystone.yaml b/prometheus-openstack-exporter/templates/secret-keystone.yaml new file mode 100644 index 0000000000..4672d68fb3 --- /dev/null +++ b/prometheus-openstack-exporter/templates/secret-keystone.yaml @@ -0,0 +1,28 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_keystone }} +{{- $envAll := . }} +{{- range $key1, $userClass := tuple "admin" "user" }} +{{- $secretName := index $envAll.Values.secrets.identity $userClass }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} +type: Opaque +data: +{{- tuple $userClass "internal" $envAll | include "helm-toolkit.snippets.keystone_secret_openrc" | indent 2 -}} +{{- end }} +{{- end }} diff --git a/prometheus-openstack-exporter/templates/secret-registry.yaml b/prometheus-openstack-exporter/templates/secret-registry.yaml new file mode 100644 index 0000000000..da979b3223 --- /dev/null +++ b/prometheus-openstack-exporter/templates/secret-registry.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_registry .Values.endpoints.oci_image_registry.auth.enabled }} +{{ include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) }} +{{- end }} diff --git a/prometheus-openstack-exporter/templates/service.yaml b/prometheus-openstack-exporter/templates/service.yaml new file mode 100644 index 0000000000..e499acf23f --- /dev/null +++ b/prometheus-openstack-exporter/templates/service.yaml @@ -0,0 +1,36 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.service }} +{{- $envAll := . }} +{{- $prometheus_annotations := $envAll.Values.monitoring.prometheus.openstack_exporter }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "prometheus_openstack_exporter" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + labels: +{{ tuple $envAll "prometheus-openstack-exporter" "metrics" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: +{{- if .Values.monitoring.prometheus.enabled }} +{{ tuple $prometheus_annotations | include "helm-toolkit.snippets.prometheus_service_annotations" | indent 4 }} +{{- end }} +spec: + ports: + - name: http + port: {{ tuple "prometheus_openstack_exporter" "internal" "exporter" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + targetPort: {{ tuple "prometheus_openstack_exporter" "internal" "exporter" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + selector: +{{ tuple $envAll "prometheus-openstack-exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- end }} diff --git a/prometheus-openstack-exporter/values.yaml b/prometheus-openstack-exporter/values.yaml new file mode 100644 index 0000000000..82dde78a6f --- /dev/null +++ b/prometheus-openstack-exporter/values.yaml @@ -0,0 +1,249 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for prometheus-openstack-exporter. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +--- +images: + tags: + prometheus_openstack_exporter: docker.io/openstackhelm/prometheus-openstack-exporter:latest-ubuntu_jammy + dep_check: quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal + image_repo_sync: docker.io/library/docker:17.07.0 + ks_user: docker.io/openstackhelm/heat:2024.1-ubuntu_jammy + pull_policy: IfNotPresent + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +labels: + openstack_exporter: + node_selector_key: openstack-control-plane + node_selector_value: enabled + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +pod: + security_context: + exporter: + pod: + runAsUser: 65534 + container: + openstack_metrics_exporter: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + ks_user: + pod: + runAsUser: 65534 + container: + prometheus_openstack_exporter_ks_user: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + mounts: + prometheus_openstack_exporter: + prometheus_openstack_exporter: + init_container: null + replicas: + prometheus_openstack_exporter: 1 + lifecycle: + upgrades: + deployments: + revision_history: 3 + pod_replacement_strategy: RollingUpdate + rolling_update: + max_unavailable: 1 + max_surge: 3 + termination_grace_period: + prometheus_openstack_exporter: + timeout: 30 + resources: + enabled: false + prometheus_openstack_exporter: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + jobs: + image_repo_sync: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + ks_user: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - prometheus-openstack-exporter-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + ks_user: + services: + - endpoint: internal + service: identity + prometheus_openstack_exporter: + jobs: + - prometheus-openstack-exporter-ks-user + services: + - endpoint: internal + service: identity + +conf: + prometheus_openstack_exporter: + OS_POLLING_INTERVAL: 30 + TIMEOUT_SECONDS: 20 + OS_RETRIES: 1 + +secrets: + identity: + admin: prometheus-openstack-exporter-keystone-admin + user: prometheus-openstack-exporter-keystone-user + oci_image_registry: + prometheus-openstack-exporter: prometheus-openstack-exporter-oci-image-registry-key + tls: + identity: + api: + # This name should be same as in keystone. Keystone + # secret will be used in these charts + # + internal: keystone-tls-api + + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + prometheus-openstack-exporter: + username: prometheus-openstack-exporter + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + prometheus_openstack_exporter: + namespace: null + hosts: + default: openstack-metrics + host_fqdn_override: + default: null + path: + default: null + scheme: + default: 'http' + port: + exporter: + default: 9103 + identity: + name: keystone + auth: + admin: + region_name: RegionOne + username: admin + password: password + project_name: admin + user_domain_name: default + project_domain_name: default + user: + role: admin + region_name: RegionOne + username: prometheus-openstack-exporter + password: password + project_name: service + user_domain_name: default + project_domain_name: default + hosts: + default: keystone + internal: keystone-api + host_fqdn_override: + default: null + path: + default: /v3 + scheme: + default: 'http' + port: + api: + default: 80 + internal: 5000 + +monitoring: + prometheus: + enabled: true + openstack_exporter: + scrape: true + +network: + openstack_metrics_exporter: + port: 9103 + +network_policy: + prometheus-openstack-exporter: + ingress: + - {} + egress: + - {} + +manifests: + certificates: false + configmap_bin: true + deployment: true + job_image_repo_sync: true + job_ks_user: true + network_policy: false + secret_keystone: true + secret_registry: true + service: true +... diff --git a/prometheus-process-exporter/Chart.yaml b/prometheus-process-exporter/Chart.yaml new file mode 100644 index 0000000000..9dfe4226f0 --- /dev/null +++ b/prometheus-process-exporter/Chart.yaml @@ -0,0 +1,29 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: v0.2.11 +description: OpenStack-Helm Process Exporter for Prometheus +name: prometheus-process-exporter +version: 2024.2.0 +home: https://github.com/openstack/openstack-helm-infra +sources: + - https://github.com/ncabatoff/process-exporter + - https://opendev.org/openstack/openstack-helm-infra +maintainers: + - name: OpenStack-Helm Authors +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit + version: ">= 0.1.0" +... diff --git a/prometheus-process-exporter/templates/daemonset.yaml b/prometheus-process-exporter/templates/daemonset.yaml new file mode 100644 index 0000000000..71f9334cbc --- /dev/null +++ b/prometheus-process-exporter/templates/daemonset.yaml @@ -0,0 +1,99 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.daemonset }} +{{- $envAll := . }} + +{{- $serviceAccountName := printf "%s-%s" .Release.Name "process-exporter" }} +{{ tuple $envAll "process_exporter" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: run-process-exporter +subjects: + - kind: ServiceAccount + name: {{ $serviceAccountName }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: cluster-admin + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: process-exporter + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "process_exporter" "metrics" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + selector: + matchLabels: +{{ tuple $envAll "process_exporter" "metrics" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} +{{ tuple $envAll "process_exporter" | include "helm-toolkit.snippets.kubernetes_upgrades_daemonset" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "process_exporter" "metrics" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ dict "envAll" $envAll "podName" "process-exporter" "containerNames" (list "process-exporter" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "metrics" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} +{{ if .Values.pod.tolerations.process_exporter.enabled }} +{{ tuple $envAll "process_exporter" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }} +{{ else }} + nodeSelector: + {{ .Values.labels.process_exporter.node_selector_key }}: {{ .Values.labels.process_exporter.node_selector_value }} +{{ end }} + hostNetwork: true + hostPID: true + initContainers: +{{ tuple $envAll "process_exporter" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: process-exporter +{{ tuple $envAll "process_exporter" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.process_exporter | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "metrics" "container" "process_exporter" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + args: +{{- if hasKey .Values.conf "children" }} + - -children={{ .Values.conf.children }} +{{- end }} + - -procnames + - {{ .Values.conf.processes }} + ports: + - name: metrics + containerPort: {{ tuple "process_exporter_metrics" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + hostPort: {{ tuple "process_exporter_metrics" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + readinessProbe: + tcpSocket: + port: {{ tuple "process_exporter_metrics" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + initialDelaySeconds: 20 + periodSeconds: 10 + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: proc + mountPath: /host/proc + readOnly: true + volumes: + - name: pod-tmp + emptyDir: {} + - name: proc + hostPath: + path: /proc +{{- end }} diff --git a/prometheus-process-exporter/templates/job-image-repo-sync.yaml b/prometheus-process-exporter/templates/job-image-repo-sync.yaml new file mode 100644 index 0000000000..08ff392992 --- /dev/null +++ b/prometheus-process-exporter/templates/job-image-repo-sync.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "process-exporter" -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/prometheus-process-exporter/templates/network_policy.yaml b/prometheus-process-exporter/templates/network_policy.yaml new file mode 100644 index 0000000000..427f71d9cd --- /dev/null +++ b/prometheus-process-exporter/templates/network_policy.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */}} + +{{- if .Values.manifests.network_policy -}} +{{- $netpol_opts := dict "envAll" . "name" "application" "label" "process_exporter" -}} +{{ $netpol_opts | include "helm-toolkit.manifests.kubernetes_network_policy" }} +{{- end -}} diff --git a/prometheus-process-exporter/templates/secret-registry.yaml b/prometheus-process-exporter/templates/secret-registry.yaml new file mode 100644 index 0000000000..da979b3223 --- /dev/null +++ b/prometheus-process-exporter/templates/secret-registry.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_registry .Values.endpoints.oci_image_registry.auth.enabled }} +{{ include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) }} +{{- end }} diff --git a/prometheus-process-exporter/templates/service.yaml b/prometheus-process-exporter/templates/service.yaml new file mode 100644 index 0000000000..ac04e22c7b --- /dev/null +++ b/prometheus-process-exporter/templates/service.yaml @@ -0,0 +1,38 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.service }} +{{- $envAll := . }} +{{- $prometheus_annotations := $envAll.Values.monitoring.prometheus.process_exporter }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "process_exporter_metrics" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + labels: +{{ tuple $envAll "process_exporter" "metrics" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: +{{- if .Values.monitoring.prometheus.enabled }} +{{ tuple $prometheus_annotations | include "helm-toolkit.snippets.prometheus_service_annotations" | indent 4 }} +{{- end }} +spec: + type: ClusterIP + clusterIP: None + ports: + - name: metrics + port: {{ tuple "process_exporter_metrics" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + targetPort: {{ tuple "process_exporter_metrics" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + selector: +{{ tuple $envAll "process_exporter" "metrics" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- end }} diff --git a/prometheus-process-exporter/values.yaml b/prometheus-process-exporter/values.yaml new file mode 100644 index 0000000000..5b95dc9681 --- /dev/null +++ b/prometheus-process-exporter/values.yaml @@ -0,0 +1,184 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for process-exporter. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +--- +images: + tags: + process_exporter: docker.io/ncabatoff/process-exporter:0.2.11 + dep_check: quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal + image_repo_sync: docker.io/library/docker:17.07.0 + pull_policy: IfNotPresent + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +labels: + process_exporter: + node_selector_key: openstack-control-plane + node_selector_value: enabled + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +pod: + security_context: + metrics: + pod: + runAsUser: 65534 + container: + process_exporter: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + weight: + default: 10 + mounts: + process_exporter: + process_exporter: + init_container: null + lifecycle: + upgrades: + daemonsets: + pod_replacement_strategy: RollingUpdate + process_exporter: + enabled: true + min_ready_seconds: 0 + max_unavailable: 1 + termination_grace_period: + process_exporter: + timeout: 30 + resources: + enabled: false + process_exporter: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + jobs: + image_repo_sync: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + tolerations: + process_exporter: + enabled: false + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + - key: node-role.kubernetes.io/control-plane + operator: Exists + - key: node-role.kubernetes.io/node + operator: Exists +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - process-exporter-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + process_exporter: + services: null + +monitoring: + prometheus: + enabled: true + process_exporter: + scrape: true + +secrets: + oci_image_registry: + prometheus-process-exporter: prometheus-process-exporter-oci-image-registry-key + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + prometheus-process-exporter: + username: prometheus-process-exporter + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + process_exporter_metrics: + namespace: null + hosts: + default: process-exporter + host_fqdn_override: + default: null + path: + default: null + scheme: + default: 'http' + port: + metrics: + default: 9256 + +network_policy: + process_exporter: + ingress: + - {} + egress: + - {} + +manifests: + configmap_bin: true + daemonset: true + job_image_repo_sync: true + secret_registry: true + service: true + +conf: + processes: dockerd,kubelet,kube-proxy,bgsagent,bgscollect,bgssd + children: true +... diff --git a/prometheus/Chart.yaml b/prometheus/Chart.yaml new file mode 100644 index 0000000000..d763da8517 --- /dev/null +++ b/prometheus/Chart.yaml @@ -0,0 +1,29 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: v2.25.0 +description: OpenStack-Helm Prometheus +name: prometheus +version: 2024.2.0 +home: https://prometheus.io/ +sources: + - https://github.com/prometheus/prometheus + - https://opendev.org/openstack/openstack-helm-infra +maintainers: + - name: OpenStack-Helm Authors +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit + version: ">= 0.1.0" +... diff --git a/prometheus/templates/bin/_apache.sh.tpl b/prometheus/templates/bin/_apache.sh.tpl new file mode 100644 index 0000000000..6e66ebc03b --- /dev/null +++ b/prometheus/templates/bin/_apache.sh.tpl @@ -0,0 +1,48 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ev + +COMMAND="${@:-start}" + +function start () { + + if [ -f /etc/apache2/envvars ]; then + # Loading Apache2 ENV variables + source /etc/httpd/apache2/envvars + fi + # Apache gets grumpy about PID files pre-existing + rm -f /etc/httpd/logs/httpd.pid + + if [ -f /usr/local/apache2/conf/.htpasswd ]; then + htpasswd -b /usr/local/apache2/conf/.htpasswd "$PROMETHEUS_ADMIN_USERNAME" "$PROMETHEUS_ADMIN_PASSWORD" + else + htpasswd -cb /usr/local/apache2/conf/.htpasswd "$PROMETHEUS_ADMIN_USERNAME" "$PROMETHEUS_ADMIN_PASSWORD" + fi + + if [ -n "$PROMETHEUS_FEDERATE_USERNAME" ]; then + htpasswd -b /usr/local/apache2/conf/.htpasswd "$PROMETHEUS_FEDERATE_USERNAME" "$PROMETHEUS_FEDERATE_PASSWORD" + fi + + #Launch Apache on Foreground + exec httpd -DFOREGROUND +} + +function stop () { + apachectl -k graceful-stop +} + +$COMMAND diff --git a/prometheus/templates/bin/_helm-tests.sh.tpl b/prometheus/templates/bin/_helm-tests.sh.tpl new file mode 100644 index 0000000000..6e736d3cbe --- /dev/null +++ b/prometheus/templates/bin/_helm-tests.sh.tpl @@ -0,0 +1,60 @@ +#!/bin/bash +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + + +set -ex + +function endpoints_up () { + endpoints_result=$(curl ${CACERT_OPTION} -K- <<< "--user ${PROMETHEUS_ADMIN_USERNAME}:${PROMETHEUS_ADMIN_PASSWORD}" \ + "${PROMETHEUS_ENDPOINT}/api/v1/query?query=up" \ + | python -c "import sys, json; print(json.load(sys.stdin)['status'])") + if [ "$endpoints_result" = "success" ]; + then + echo "PASS: Endpoints successfully queried!" + else + echo "FAIL: Endpoints not queried!"; + exit 1; + fi +} + +function get_targets () { + targets_result=$(curl ${CACERT_OPTION} -K- <<< "--user ${PROMETHEUS_ADMIN_USERNAME}:${PROMETHEUS_ADMIN_PASSWORD}" \ + "${PROMETHEUS_ENDPOINT}/api/v1/targets" \ + | python -c "import sys, json; print(json.load(sys.stdin)['status'])") + if [ "$targets_result" = "success" ]; + then + echo "PASS: Targets successfully queried!" + else + echo "FAIL: Endpoints not queried!"; + exit 1; + fi +} + +function get_alertmanagers () { + alertmanager=$(curl ${CACERT_OPTION} -K- <<< "--user ${PROMETHEUS_ADMIN_USERNAME}:${PROMETHEUS_ADMIN_PASSWORD}" \ + "${PROMETHEUS_ENDPOINT}/api/v1/alertmanagers" \ + | python -c "import sys, json; print(json.load(sys.stdin)['status'])") + if [ "$alertmanager" = "success" ]; + then + echo "PASS: Alertmanager successfully queried!" + else + echo "FAIL: Alertmanager not queried!"; + exit 1; + fi +} + +endpoints_up +get_targets +get_alertmanagers diff --git a/prometheus/templates/bin/_prometheus.sh.tpl b/prometheus/templates/bin/_prometheus.sh.tpl new file mode 100644 index 0000000000..25cb905286 --- /dev/null +++ b/prometheus/templates/bin/_prometheus.sh.tpl @@ -0,0 +1,37 @@ +#!/bin/sh + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +# Two ways how to launch init process in container: by default and custom (defined in override values). +{{ $deflaunch := .Values.proc_launch.prometheus.default }} +if [ "{{ $deflaunch }}" = true ] +then + COMMAND="${@:-start}" + + function start () { + {{ $flags := include "prometheus.utils.command_line_flags" .Values.conf.prometheus.command_line_flags }} + exec /bin/prometheus --config.file=/etc/config/prometheus.yml {{ $flags }} + } + + function stop () { + kill -TERM 1 + } + + $COMMAND +else + {{ tpl (.Values.proc_launch.prometheus.custom_launch) . }} +fi diff --git a/prometheus/templates/certificates.yaml b/prometheus/templates/certificates.yaml new file mode 100644 index 0000000000..40b5aa709e --- /dev/null +++ b/prometheus/templates/certificates.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.certificates -}} +{{ dict "envAll" . "service" "monitoring" "type" "internal" | include "helm-toolkit.manifests.certificates" }} +{{- end -}} diff --git a/prometheus/templates/configmap-bin.yaml b/prometheus/templates/configmap-bin.yaml new file mode 100644 index 0000000000..a907f291e6 --- /dev/null +++ b/prometheus/templates/configmap-bin.yaml @@ -0,0 +1,31 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_bin }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ printf "%s-%s" $envAll.Release.Name "prometheus-bin" | quote }} +data: + apache.sh: | +{{ tuple "bin/_apache.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + prometheus.sh: | +{{ tuple "bin/_prometheus.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + helm-tests.sh: | +{{ tuple "bin/_helm-tests.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + image-repo-sync.sh: | +{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} +{{- end }} diff --git a/prometheus/templates/configmap-etc.yaml b/prometheus/templates/configmap-etc.yaml new file mode 100644 index 0000000000..b5e36191b1 --- /dev/null +++ b/prometheus/templates/configmap-etc.yaml @@ -0,0 +1,30 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_etc }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ printf "%s-%s" $envAll.Release.Name "prometheus-etc" | quote }} +type: Opaque +data: +{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.prometheus.scrape_configs.template "key" "prometheus.yml" "format" "Secret") | indent 2 }} +{{ range $key, $value := .Values.conf.prometheus.rules }} + {{ $key }}.rules: {{ toYaml $value | b64enc }} +{{ end }} + # NOTE(srwilkers): this must be last, to work round helm ~2.7 bug. +{{- include "helm-toolkit.snippets.values_template_renderer" (dict "envAll" $envAll "template" .Values.conf.httpd "key" "httpd.conf" "format" "Secret") | indent 2 }} +{{- end }} diff --git a/prometheus/templates/ingress-prometheus.yaml b/prometheus/templates/ingress-prometheus.yaml new file mode 100644 index 0000000000..60b928407d --- /dev/null +++ b/prometheus/templates/ingress-prometheus.yaml @@ -0,0 +1,24 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.ingress .Values.network.prometheus.ingress.public }} +{{- $envAll := . -}} +{{- $port := tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" }} +{{- $ingressOpts := dict "envAll" $envAll "backendService" "prometheus" "backendServiceType" "monitoring" "backendPort" $port -}} +{{- $secretName := $envAll.Values.secrets.tls.monitoring.prometheus.internal -}} +{{- if and .Values.manifests.certificates $secretName -}} +{{- $_ := set $ingressOpts "certIssuer" .Values.endpoints.monitoring.host_fqdn_override.default.tls.issuerRef.name -}} +{{- end -}} +{{ $ingressOpts | include "helm-toolkit.manifests.ingress" }} +{{- end }} diff --git a/prometheus/templates/job-image-repo-sync.yaml b/prometheus/templates/job-image-repo-sync.yaml new file mode 100644 index 0000000000..661b284df1 --- /dev/null +++ b/prometheus/templates/job-image-repo-sync.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "prometheus" -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/prometheus/templates/network_policy.yaml b/prometheus/templates/network_policy.yaml new file mode 100644 index 0000000000..2b7bc8bdca --- /dev/null +++ b/prometheus/templates/network_policy.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. */}} + +{{- if .Values.manifests.network_policy -}} +{{- $netpol_opts := dict "envAll" . "name" "application" "label" "prometheus" -}} +{{ $netpol_opts | include "helm-toolkit.manifests.kubernetes_network_policy" }} +{{- end -}} diff --git a/prometheus/templates/pod-helm-tests.yaml b/prometheus/templates/pod-helm-tests.yaml new file mode 100644 index 0000000000..0549b64c44 --- /dev/null +++ b/prometheus/templates/pod-helm-tests.yaml @@ -0,0 +1,80 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.helm_tests }} +{{- $envAll := . }} + +{{- $serviceAccountName := print .Release.Name "-test" }} +{{ tuple $envAll "tests" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: v1 +kind: Pod +metadata: + name: "{{.Release.Name}}-test" + labels: +{{ tuple $envAll "prometheus" "test" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +{{ dict "envAll" $envAll "podName" "prometheus-test" "containerNames" (list "init" "prometheus-helm-tests") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 4 }} + "helm.sh/hook": test-success +spec: +{{ dict "envAll" $envAll "application" "test" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 2 }} + serviceAccountName: {{ $serviceAccountName }} + nodeSelector: + {{ .Values.labels.test.node_selector_key }}: {{ .Values.labels.test.node_selector_value }} + restartPolicy: Never + initContainers: +{{ tuple $envAll "tests" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 4 }} + containers: + - name: prometheus-helm-tests +{{ tuple $envAll "helm_tests" | include "helm-toolkit.snippets.image" | indent 6 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.tests | include "helm-toolkit.snippets.kubernetes_resources" | indent 6 }} +{{ dict "envAll" $envAll "application" "test" "container" "prometheus_helm_tests" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 6 }} + command: + - /tmp/helm-tests.sh + env: + - name: PROMETHEUS_ADMIN_USERNAME + valueFrom: + secretKeyRef: + name: {{ printf "%s-%s" $envAll.Release.Name "admin-user" | quote }} + key: PROMETHEUS_ADMIN_USERNAME + - name: PROMETHEUS_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: {{ printf "%s-%s" $envAll.Release.Name "admin-user" | quote }} + key: PROMETHEUS_ADMIN_PASSWORD + +{{- if .Values.manifests.certificates }} + - name: CACERT_OPTION + value: "--cacert /etc/prometheus/certs/ca.crt" +{{- end }} + - name: PROMETHEUS_ENDPOINT + value: {{ printf "%s://%s" (tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup") (tuple "monitoring" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup") }} + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: prometheus-bin + mountPath: /tmp/helm-tests.sh + subPath: helm-tests.sh + readOnly: true +{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.monitoring.prometheus.internal "path" "/etc/prometheus/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 8 }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: prometheus-bin + configMap: + name: {{ printf "%s-%s" $envAll.Release.Name "prometheus-bin" | quote }} + defaultMode: 0555 +{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.monitoring.prometheus.internal | include "helm-toolkit.snippets.tls_volume" | indent 4 }} +{{- end }} diff --git a/prometheus/templates/secret-ingress-tls.yaml b/prometheus/templates/secret-ingress-tls.yaml new file mode 100644 index 0000000000..efffee60b4 --- /dev/null +++ b/prometheus/templates/secret-ingress-tls.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_ingress_tls }} +{{- include "helm-toolkit.manifests.secret_ingress_tls" ( dict "envAll" . "backendServiceType" "monitoring" "backendService" "prometheus" ) }} +{{- end }} diff --git a/prometheus/templates/secret-prometheus.yaml b/prometheus/templates/secret-prometheus.yaml new file mode 100644 index 0000000000..ac856d3a8a --- /dev/null +++ b/prometheus/templates/secret-prometheus.yaml @@ -0,0 +1,28 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_prometheus }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ printf "%s-%s" $envAll.Release.Name "admin-user" | quote }} +type: Opaque +data: + PROMETHEUS_ADMIN_USERNAME: {{ .Values.endpoints.monitoring.auth.admin.username | b64enc }} + PROMETHEUS_ADMIN_PASSWORD: {{ .Values.endpoints.monitoring.auth.admin.password | b64enc }} + PROMETHEUS_FEDERATE_USERNAME: {{ .Values.endpoints.monitoring.auth.federate.username | b64enc }} + PROMETHEUS_FEDERATE_PASSWORD: {{ .Values.endpoints.monitoring.auth.federate.password | b64enc }} +{{- end }} diff --git a/prometheus/templates/secret-registry.yaml b/prometheus/templates/secret-registry.yaml new file mode 100644 index 0000000000..da979b3223 --- /dev/null +++ b/prometheus/templates/secret-registry.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_registry .Values.endpoints.oci_image_registry.auth.enabled }} +{{ include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) }} +{{- end }} diff --git a/prometheus/templates/secret-tls-configs.yaml b/prometheus/templates/secret-tls-configs.yaml new file mode 100644 index 0000000000..40a86a840a --- /dev/null +++ b/prometheus/templates/secret-tls-configs.yaml @@ -0,0 +1,27 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.tls_configs }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Release.Name }}-tls-configs +data: +{{- range $k, $v := .Values.tls_configs }} +{{- range $f, $c := $v }} + {{ $k }}.{{ $f }}: {{ $c | b64enc }} +{{- end }} +{{- end }} +{{- end }} diff --git a/prometheus/templates/service-ingress-prometheus.yaml b/prometheus/templates/service-ingress-prometheus.yaml new file mode 100644 index 0000000000..2bfa2e402e --- /dev/null +++ b/prometheus/templates/service-ingress-prometheus.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.service_ingress .Values.network.prometheus.ingress.public }} +{{- $serviceIngressOpts := dict "envAll" . "backendServiceType" "monitoring" -}} +{{ $serviceIngressOpts | include "helm-toolkit.manifests.service_ingress" }} +{{- end }} diff --git a/prometheus/templates/service.yaml b/prometheus/templates/service.yaml new file mode 100644 index 0000000000..d1df7eec43 --- /dev/null +++ b/prometheus/templates/service.yaml @@ -0,0 +1,42 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.service }} +{{- $envAll := . }} +{{- $prometheus_annotations := $envAll.Values.monitoring.prometheus.prometheus }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "monitoring" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + labels: +{{ tuple $envAll "prometheus" "metrics" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: +{{- if .Values.monitoring.prometheus.enabled }} +{{ tuple $prometheus_annotations | include "helm-toolkit.snippets.prometheus_service_annotations" | indent 4 }} +{{- end }} +spec: + ports: + - name: {{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" }} + port: {{ tuple "monitoring" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + targetPort: {{ tuple "monitoring" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + {{ if .Values.network.prometheus.node_port.enabled }} + nodePort: {{ .Values.network.prometheus.node_port.port }} + {{ end }} + selector: +{{ tuple $envAll "prometheus" "api" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + {{ if .Values.network.prometheus.node_port.enabled }} + type: NodePort + {{ end }} +{{- end }} diff --git a/prometheus/templates/statefulset.yaml b/prometheus/templates/statefulset.yaml new file mode 100644 index 0000000000..d624dd4571 --- /dev/null +++ b/prometheus/templates/statefulset.yaml @@ -0,0 +1,261 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "probeTemplate" }} +{{- $probePort := tuple "monitoring" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +{{- $probeUser := .Values.endpoints.monitoring.auth.admin.username }} +{{- $probePass := .Values.endpoints.monitoring.auth.admin.password }} +{{- $authHeader := printf "%s:%s" $probeUser $probePass | b64enc }} +httpGet: + path: /-/ready + scheme: {{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" | upper }} + port: {{ $probePort }} + httpHeaders: + - name: Authorization + value: Basic {{ $authHeader }} +{{- end }} + + +{{- if .Values.manifests.statefulset_prometheus }} +{{- $envAll := . }} + +{{- $mounts_prometheus := .Values.pod.mounts.prometheus.prometheus }} +{{- $mounts_prometheus_init := .Values.pod.mounts.prometheus.init_container }} + +{{- $rcControllerName := printf "%s-%s" $envAll.Release.Name "prometheus" }} +{{ tuple $envAll "prometheus" $rcControllerName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ $rcControllerName | quote }} +rules: + - apiGroups: + - "" + resources: + - nodes + - nodes/proxy + - services + - endpoints + - pods + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - nonResourceURLs: + - "/metrics" + verbs: + - get +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ $rcControllerName | quote }} +subjects: + - kind: ServiceAccount + name: {{ $rcControllerName | quote }} + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ $rcControllerName | quote }} + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ $rcControllerName | quote }} + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "prometheus" "api" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + serviceName: {{ tuple "monitoring" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + podManagementPolicy: "Parallel" + replicas: {{ .Values.pod.replicas.prometheus }} + selector: + matchLabels: +{{ tuple $envAll "prometheus" "api" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} + template: + metadata: + labels: +{{ tuple $envAll "prometheus" "api" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} +{{ dict "envAll" $envAll "podName" "prometheus" "containerNames" (list "prometheus" "prometheus-perms" "apache-proxy" "init") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "api" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $rcControllerName | quote }} + affinity: +{{ tuple $envAll "prometheus" "api" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} + nodeSelector: + {{ .Values.labels.prometheus.node_selector_key }}: {{ .Values.labels.prometheus.node_selector_value | quote }} + terminationGracePeriodSeconds: {{ .Values.pod.lifecycle.termination_grace_period.prometheus.timeout | default "30" }} + initContainers: +{{ tuple $envAll "prometheus" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + - name: prometheus-perms +{{ tuple $envAll "prometheus" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.prometheus | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "api" "container" "prometheus_perms" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - chown + - -R + - "nobody:" + - /var/lib/prometheus/data + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: storage + mountPath: /var/lib/prometheus/data + containers: + - name: apache-proxy +{{ tuple $envAll "apache_proxy" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.apache_proxy | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "api" "container" "apache_proxy" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/apache.sh + - start + ports: + - name: {{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_scheme_lookup" }} + containerPort: {{ tuple "monitoring" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + env: + - name: PROMETHEUS_PORT + value: {{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: PROMETHEUS_ADMIN_USERNAME + valueFrom: + secretKeyRef: + name: {{ printf "%s-%s" $envAll.Release.Name "admin-user" | quote }} + key: PROMETHEUS_ADMIN_USERNAME + - name: PROMETHEUS_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: {{ printf "%s-%s" $envAll.Release.Name "admin-user" | quote }} + key: PROMETHEUS_ADMIN_PASSWORD + - name: PROMETHEUS_FEDERATE_USERNAME + valueFrom: + secretKeyRef: + name: {{ printf "%s-%s" $envAll.Release.Name "admin-user" | quote }} + key: PROMETHEUS_FEDERATE_USERNAME + - name: PROMETHEUS_FEDERATE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ printf "%s-%s" $envAll.Release.Name "admin-user" | quote }} + key: PROMETHEUS_FEDERATE_PASSWORD + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: prometheus-bin + mountPath: /tmp/apache.sh + subPath: apache.sh + readOnly: true + - name: prometheus-etc + mountPath: /usr/local/apache2/conf/httpd.conf + subPath: httpd.conf + readOnly: true +{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.monitoring.prometheus.internal "path" "/etc/prometheus/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }} + - name: prometheus +{{ tuple $envAll "prometheus" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.prometheus | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "api" "container" "prometheus" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/prometheus.sh + - start + lifecycle: + preStop: + exec: + command: + - /tmp/prometheus.sh + - stop + ports: + - name: prom-metrics + containerPort: {{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +{{ dict "envAll" . "component" "prometheus" "container" "prometheus" "type" "readiness" "probeTemplate" (include "probeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }} +{{ dict "envAll" . "component" "prometheus" "container" "prometheus" "type" "liveness" "probeTemplate" (include "probeTemplate" . | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | indent 10 }} + env: +{{- if .Values.pod.env.prometheus }} +{{ include "helm-toolkit.utils.to_k8s_env_vars" .Values.pod.env.prometheus | indent 12 }} +{{- end }} + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: etcprometheus + mountPath: /etc/config + - name: rulesprometheus + mountPath: /etc/config/rules + {{- range $key, $value := .Values.conf.prometheus.rules }} + - name: prometheus-etc + mountPath: /etc/config/rules/{{ $key }}.rules + subPath: {{ $key }}.rules + readOnly: true + {{- end }} + - name: prometheus-etc + mountPath: /etc/config/prometheus.yml + subPath: prometheus.yml + readOnly: true + - name: prometheus-bin + mountPath: /tmp/prometheus.sh + subPath: prometheus.sh + readOnly: true + - name: storage + mountPath: /var/lib/prometheus/data +{{- if .Values.tls_configs }} + - name: tls-configs + mountPath: /tls_configs +{{- end }} +{{ if $mounts_prometheus.volumeMounts }}{{ toYaml $mounts_prometheus.volumeMounts | indent 12 }}{{ end }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: etcprometheus + emptyDir: {} + - name: rulesprometheus + emptyDir: {} + - name: prometheus-etc + secret: + secretName: {{ printf "%s-%s" $envAll.Release.Name "prometheus-etc" | quote }} + defaultMode: 0444 +{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.monitoring.prometheus.internal | include "helm-toolkit.snippets.tls_volume" | indent 8 }} + - name: prometheus-bin + configMap: + name: {{ printf "%s-%s" $envAll.Release.Name "prometheus-bin" | quote }} + defaultMode: 0555 +{{- if .Values.tls_configs }} + - name: tls-configs + secret: + secretName: {{ printf "%s-%s" $envAll.Release.Name "tls-configs" | quote }} + defaultMode: 0444 +{{- end }} +{{ if $mounts_prometheus.volumes }}{{ toYaml $mounts_prometheus.volumes | indent 8 }}{{ end }} +{{- if not .Values.storage.enabled }} + - name: storage + emptyDir: {} +{{- else }} + volumeClaimTemplates: + - metadata: + name: storage + spec: + accessModes: {{ .Values.storage.pvc.access_mode }} + resources: + requests: + storage: {{ .Values.storage.requests.storage }} + storageClassName: {{ .Values.storage.storage_class }} +{{- end }} +{{- end }} diff --git a/prometheus/templates/utils/_command_line_flags.tpl b/prometheus/templates/utils/_command_line_flags.tpl new file mode 100644 index 0000000000..229fae2664 --- /dev/null +++ b/prometheus/templates/utils/_command_line_flags.tpl @@ -0,0 +1,46 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +# This function generates the command line flags passed to Prometheus at time of +# execution. This allows the Prometheus service configuration to be flexible, as +# the only way to define Prometheus's configuration is via command line flags. +# The yaml definition for these flags uses the full yaml path as the key, and +# replaces underscores with hyphens to match the syntax required for the flags +# generated (This is required due to Go's yaml parsing capabilities). +# For example: +# +# conf: +# prometheus: +# command_line_flags: +# storage.tsdb.max_block_duration: 2h +# +# Will generate the following flag: +# --storage.tsdb.max-block-duration=2h +# +# Prometheus's command flags can be found by either running 'prometheus -h' or +# 'prometheus --help-man' + +{{- define "prometheus.utils.command_line_flags" -}} +{{- range $flag, $value := . -}} +{{- $flag := $flag | replace "_" "-" }} +{{- if eq $flag "web.enable-admin-api" "web.enable-lifecycle" "storage.tsdb.wal-compression" -}} +{{- if $value }} +{{- printf " --%s " $flag -}} +{{- end -}} +{{- else -}} +{{- $value := $value | toString }} +{{- printf " --%s=%s " $flag $value }} +{{- end -}} +{{- end -}} +{{- end -}} diff --git a/prometheus/values.yaml b/prometheus/values.yaml new file mode 100644 index 0000000000..ccac6f12fd --- /dev/null +++ b/prometheus/values.yaml @@ -0,0 +1,1136 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for prometheus. +# This is a YAML-formatted file. +# Declare name/value pairs to be passed into your templates. +# name: value + +--- +images: + tags: + apache_proxy: docker.io/library/httpd:2.4 + prometheus: docker.io/prom/prometheus:v2.25.0 + helm_tests: docker.io/openstackhelm/heat:wallaby-ubuntu_focal + dep_check: quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal + image_repo_sync: docker.io/library/docker:17.07.0 + pull_policy: IfNotPresent + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +labels: + prometheus: + node_selector_key: openstack-control-plane + node_selector_value: enabled + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + test: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +pod: + env: + prometheus: null + security_context: + api: + pod: + runAsUser: 65534 + container: + prometheus_perms: + runAsUser: 0 + readOnlyRootFilesystem: false + apache_proxy: + runAsUser: 0 + readOnlyRootFilesystem: false + prometheus: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + test: + pod: + runAsUser: 65534 + container: + prometheus_helm_tests: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + weight: + default: 10 + mounts: + prometheus: + prometheus: + init_container: null + replicas: + prometheus: 1 + lifecycle: + upgrades: + statefulsets: + pod_replacement_strategy: RollingUpdate + termination_grace_period: + prometheus: + timeout: 30 + resources: + enabled: false + prometheus: + limits: + memory: "1024Mi" + cpu: "2000m" + requests: + memory: "128Mi" + cpu: "500m" + jobs: + image_repo_sync: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + tests: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + probes: + prometheus: + prometheus: + readiness: + enabled: true + params: + initialDelaySeconds: 30 + timeoutSeconds: 30 + liveness: + enabled: false + params: + initialDelaySeconds: 120 + timeoutSeconds: 30 +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + prometheus: + username: prometheus + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + monitoring: + name: prometheus + namespace: null + auth: + admin: + username: admin + password: changeme + federate: + username: federate + password: changeme + hosts: + default: prom-metrics + public: prometheus + host_fqdn_override: + default: null + # NOTE(srwilkers): this chart supports TLS for fqdn over-ridden public + # endpoints using the following format: + # public: + # host: null + # tls: + # crt: null + # key: null + path: + default: null + scheme: + default: 'http' + port: + api: + default: 9090 + http: + default: 80 + alertmanager: + name: prometheus-alertmanager + namespace: null + hosts: + default: alerts-engine + public: prometheus-alertmanager + discovery: prometheus-alertmanager-discovery + host_fqdn_override: + default: null + path: + default: null + scheme: + default: 'http' + port: + api: + default: 9093 + public: 80 + mesh: + default: 9094 + ldap: + hosts: + default: ldap + auth: + admin: + bind: "cn=admin,dc=cluster,dc=local" + password: password + host_fqdn_override: + default: null + path: + default: "/ou=People,dc=cluster,dc=local" + scheme: + default: ldap + port: + ldap: + default: 389 + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - prometheus-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + prometheus: + services: null + tests: + services: + - endpoint: internal + service: monitoring + +monitoring: + prometheus: + enabled: true + prometheus: + scrape: true + +network: + prometheus: + ingress: + public: true + classes: + namespace: "nginx" + cluster: "nginx-cluster" + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / + nginx.ingress.kubernetes.io/affinity: cookie + nginx.ingress.kubernetes.io/session-cookie-name: kube-ingress-session-prometheus + nginx.ingress.kubernetes.io/session-cookie-hash: sha1 + nginx.ingress.kubernetes.io/session-cookie-expires: "600" + nginx.ingress.kubernetes.io/session-cookie-max-age: "600" + node_port: + enabled: false + port: 30900 + +network_policy: + prometheus: + ingress: + - {} + egress: + - {} + +proc_launch: + prometheus: + default: true + custom_launch: | + while true + do + echo "If 'proc_launch.prometheus.default: false'." + echo "Your custom shell script code you can put here." + sleep 10 + done + +secrets: + oci_image_registry: + prometheus: prometheus-oci-image-registry-key + tls: + monitoring: + prometheus: + public: prometheus-tls-public + internal: prometheus-tls-api + +tls_configs: + # If client certificates are required to connect to metrics endpoints, they + # can be configured here. They will be mounted in the pod under /tls_configs + # and can be referenced in scrape configs. + # The filenames will be the key and subkey concatenanted with a ".", e.g.: + # /tls_configs/kubernetes-etcd.ca.pem + # /tls_configs/kubernetes-etcd.crt.pem + # /tls_configs/kubernetes-etcd.key.pem + # From the following: + # kubernetes-etcd: + # ca.pem: | + # -----BEGIN CERTIFICATE----- + # -----END CERTIFICATE----- + # crt.pem: | + # -----BEGIN CERTIFICATE----- + # -----END CERTIFICATE----- + # key.pem: | + # -----BEGIN RSA PRIVATE KEY----- + # -----END RSA PRIVATE KEY----- + +storage: + enabled: true + pvc: + name: prometheus-pvc + access_mode: ["ReadWriteOnce"] + requests: + storage: 5Gi + storage_class: general + +manifests: + certificates: false + configmap_bin: true + configmap_etc: true + ingress: true + helm_tests: true + job_image_repo_sync: true + network_policy: true + secret_ingress_tls: true + secret_prometheus: true + secret_registry: true + service_ingress: true + service: true + statefulset_prometheus: true + +conf: + httpd: | + ServerRoot "/usr/local/apache2" + + Listen 80 + + LoadModule mpm_event_module modules/mod_mpm_event.so + LoadModule authn_file_module modules/mod_authn_file.so + LoadModule authn_core_module modules/mod_authn_core.so + LoadModule authz_host_module modules/mod_authz_host.so + LoadModule authz_groupfile_module modules/mod_authz_groupfile.so + LoadModule authz_user_module modules/mod_authz_user.so + LoadModule authz_core_module modules/mod_authz_core.so + LoadModule access_compat_module modules/mod_access_compat.so + LoadModule auth_basic_module modules/mod_auth_basic.so + LoadModule ldap_module modules/mod_ldap.so + LoadModule authnz_ldap_module modules/mod_authnz_ldap.so + LoadModule reqtimeout_module modules/mod_reqtimeout.so + LoadModule filter_module modules/mod_filter.so + LoadModule proxy_html_module modules/mod_proxy_html.so + LoadModule log_config_module modules/mod_log_config.so + LoadModule env_module modules/mod_env.so + LoadModule headers_module modules/mod_headers.so + LoadModule setenvif_module modules/mod_setenvif.so + LoadModule version_module modules/mod_version.so + LoadModule proxy_module modules/mod_proxy.so + LoadModule proxy_connect_module modules/mod_proxy_connect.so + LoadModule proxy_http_module modules/mod_proxy_http.so + LoadModule proxy_balancer_module modules/mod_proxy_balancer.so + LoadModule slotmem_shm_module modules/mod_slotmem_shm.so + LoadModule slotmem_plain_module modules/mod_slotmem_plain.so + LoadModule unixd_module modules/mod_unixd.so + LoadModule status_module modules/mod_status.so + LoadModule autoindex_module modules/mod_autoindex.so + + + User daemon + Group daemon + + + + AllowOverride none + Require all denied + + + + Require all denied + + + ErrorLog /dev/stderr + + LogLevel warn + + + LogFormat "%a %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined + LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" proxy + LogFormat "%h %l %u %t \"%r\" %>s %b" common + + + LogFormat "%a %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combinedio + + + SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded + CustomLog /dev/stdout common + CustomLog /dev/stdout combined + CustomLog /dev/stdout proxy env=forwarded + + + + AllowOverride None + Options None + Require all granted + + + + RequestHeader unset Proxy early + + + + Include conf/extra/proxy-html.conf + + + + # Expose metrics to all users, as this is not sensitive information and + # circumvents the inability of Prometheus to interpolate environment vars + # in its configuration file + + ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/metrics + ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/metrics + Satisfy Any + Allow from all + + # Expose the /federate endpoint to all users, as this is also not + # sensitive information and circumvents the inability of Prometheus to + # interpolate environment vars in its configuration file + + ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/metrics + ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/metrics + Satisfy Any + Allow from all + + # Restrict general user (LDAP) access to the /graph endpoint, as general trusted + # users should only be able to query Prometheus for metrics and not have access + # to information like targets, configuration, flags or build info for Prometheus + + ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/ + ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/ + AuthName "Prometheus" + AuthType Basic + AuthBasicProvider file ldap + AuthUserFile /usr/local/apache2/conf/.htpasswd + AuthLDAPBindDN {{ .Values.endpoints.ldap.auth.admin.bind }} + AuthLDAPBindPassword {{ .Values.endpoints.ldap.auth.admin.password }} + AuthLDAPURL {{ tuple "ldap" "default" "ldap" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | quote }} + Require valid-user + + + ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/graph + ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/graph + AuthName "Prometheus" + AuthType Basic + AuthBasicProvider file ldap + AuthUserFile /usr/local/apache2/conf/.htpasswd + AuthLDAPBindDN {{ .Values.endpoints.ldap.auth.admin.bind }} + AuthLDAPBindPassword {{ .Values.endpoints.ldap.auth.admin.password }} + AuthLDAPURL {{ tuple "ldap" "default" "ldap" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | quote }} + Require valid-user + + # Restrict access to the /config (dashboard) and /api/v1/status/config (http) endpoints + # to the admin user + + ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/config + ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/config + AuthName "Prometheus" + AuthType Basic + AuthBasicProvider file + AuthUserFile /usr/local/apache2/conf/.htpasswd + Require valid-user + + + ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/api/v1/status/config + ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/api/v1/status/config + AuthName "Prometheus" + AuthType Basic + AuthBasicProvider file + AuthUserFile /usr/local/apache2/conf/.htpasswd + Require valid-user + + # Restrict access to the /flags (dashboard) and /api/v1/status/flags (http) endpoints + # to the admin user + + ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/flags + ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/flags + AuthName "Prometheus" + AuthType Basic + AuthBasicProvider file + AuthUserFile /usr/local/apache2/conf/.htpasswd + Require valid-user + + + ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/api/v1/status/flags + ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/api/v1/status/flags + AuthName "Prometheus" + AuthType Basic + AuthBasicProvider file + AuthUserFile /usr/local/apache2/conf/.htpasswd + Require valid-user + + # Restrict access to the /status (dashboard) endpoint to the admin user + + ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/status + ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/status + AuthName "Prometheus" + AuthType Basic + AuthBasicProvider file + AuthUserFile /usr/local/apache2/conf/.htpasswd + Require valid-user + + # Restrict access to the /rules (dashboard) endpoint to the admin user + + ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/rules + ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/rules + AuthName "Prometheus" + AuthType Basic + AuthBasicProvider file + AuthUserFile /usr/local/apache2/conf/.htpasswd + Require valid-user + + # Restrict access to the /targets (dashboard) and /api/v1/targets (http) endpoints + # to the admin user + + ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/targets + ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/targets + AuthName "Prometheus" + AuthType Basic + AuthBasicProvider file + AuthUserFile /usr/local/apache2/conf/.htpasswd + Require valid-user + + + ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/api/v1/targets + ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/api/v1/targets + AuthName "Prometheus" + AuthType Basic + AuthBasicProvider file + AuthUserFile /usr/local/apache2/conf/.htpasswd + Require valid-user + + # Restrict access to the /api/v1/admin/tsdb/ endpoints (http) to the admin user. + # These endpoints are disabled by default, but are included here to ensure only + # an admin user has access to these endpoints when enabled + + ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/api/v1/admin/tsdb/ + ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/api/v1/admin/tsdb/ + AuthName "Prometheus" + AuthType Basic + AuthBasicProvider file + AuthUserFile /usr/local/apache2/conf/.htpasswd + Require valid-user + + + prometheus: + # Consumed by a prometheus helper function to generate the command line flags + # for configuring the prometheus service + command_line_flags: + log.level: info + query.max_concurrency: 20 + query.timeout: 2m + storage.tsdb.path: /var/lib/prometheus/data + storage.tsdb.retention.time: 7d + # NOTE(srwilkers): These settings default to false, but they are + # exposed here to allow enabling if desired. Please note the security + # impacts of enabling these flags. More information regarding the impacts + # can be found here: https://prometheus.io/docs/operating/security/ + # + # If set to true, all administrative functionality is exposed via the http + # /api/*/admin/ path + web.enable_admin_api: false + # If set to true, allows for http reloads and shutdown of Prometheus + web.enable_lifecycle: false + scrape_configs: + template: | + {{- $promHost := tuple "monitoring" "public" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }} + {{- if not (empty .Values.conf.prometheus.rules)}} + rule_files: + {{- $rulesKeys := keys .Values.conf.prometheus.rules -}} + {{- range $rule := $rulesKeys }} + {{ printf "- /etc/config/rules/%s.rules" $rule }} + {{- end }} + {{- end }} + global: + scrape_interval: 60s + evaluation_interval: 60s + external_labels: + prometheus_host: {{$promHost}} + scrape_configs: + - job_name: kubelet + scheme: https + # This TLS & bearer token file config is used to connect to the actual scrape + # endpoints for cluster components. This is separate to discovery auth + # configuration because discovery & scraping are two separate concerns in + # Prometheus. The discovery auth config is automatic if Prometheus runs inside + # the cluster. Otherwise, more config options have to be provided within the + # . + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + kubernetes_sd_configs: + - role: node + scrape_interval: 45s + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: + - __meta_kubernetes_node_name + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/${1}/proxy/metrics + - source_labels: + - __meta_kubernetes_node_name + action: replace + target_label: kubernetes_io_hostname + # Scrape config for Kubelet cAdvisor. + # + # This is required for Kubernetes 1.7.3 and later, where cAdvisor metrics + # (those whose names begin with 'container_') have been removed from the + # Kubelet metrics endpoint. This job scrapes the cAdvisor endpoint to + # retrieve those metrics. + # + # In Kubernetes 1.7.0-1.7.2, these metrics are only exposed on the cAdvisor + # HTTP endpoint; use "replacement: /api/v1/nodes/${1}:4194/proxy/metrics" + # in that case (and ensure cAdvisor's HTTP server hasn't been disabled with + # the --cadvisor-port=0 Kubelet flag). + # + # This job is not necessary and should be removed in Kubernetes 1.6 and + # earlier versions, or it will cause the metrics to be scraped twice. + - job_name: 'kubernetes-cadvisor' + + # Default to scraping over https. If required, just disable this or change to + # `http`. + scheme: https + + # This TLS & bearer token file config is used to connect to the actual scrape + # endpoints for cluster components. This is separate to discovery auth + # configuration because discovery & scraping are two separate concerns in + # Prometheus. The discovery auth config is automatic if Prometheus runs inside + # the cluster. Otherwise, more config options have to be provided within the + # . + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + + kubernetes_sd_configs: + - role: node + + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_node_label_(.+) + - target_label: __address__ + replacement: kubernetes.default.svc:443 + - source_labels: + - __meta_kubernetes_node_name + regex: (.+) + target_label: __metrics_path__ + replacement: /api/v1/nodes/${1}/proxy/metrics/cadvisor + metric_relabel_configs: + - source_labels: + - __name__ + regex: 'container_network_tcp_usage_total' + action: drop + - source_labels: + - __name__ + regex: 'container_tasks_state' + action: drop + - source_labels: + - __name__ + regex: 'container_network_udp_usage_total' + action: drop + - source_labels: + - __name__ + regex: 'container_memory_failures_total' + action: drop + - source_labels: + - __name__ + regex: 'container_cpu_load_average_10s' + action: drop + - source_labels: + - __name__ + regex: 'container_cpu_system_seconds_total' + action: drop + - source_labels: + - __name__ + regex: 'container_cpu_user_seconds_total' + action: drop + - source_labels: + - __name__ + regex: 'container_fs_inodes_free' + action: drop + - source_labels: + - __name__ + regex: 'container_fs_inodes_total' + action: drop + - source_labels: + - __name__ + regex: 'container_fs_io_current' + action: drop + - source_labels: + - __name__ + regex: 'container_fs_io_time_seconds_total' + action: drop + - source_labels: + - __name__ + regex: 'container_fs_io_time_weighted_seconds_total' + action: drop + - source_labels: + - __name__ + regex: 'container_fs_read_seconds_total' + action: drop + - source_labels: + - __name__ + regex: 'container_fs_reads_merged_total' + action: drop + - source_labels: + - __name__ + regex: 'container_fs_reads_merged_total' + action: drop + - source_labels: + - __name__ + regex: 'container_fs_reads_total' + action: drop + - source_labels: + - __name__ + regex: 'container_fs_sector_reads_total' + action: drop + - source_labels: + - __name__ + regex: 'container_fs_sector_writes_total' + action: drop + - source_labels: + - __name__ + regex: 'container_fs_write_seconds_total' + action: drop + - source_labels: + - __name__ + regex: 'container_fs_writes_bytes_total' + action: drop + - source_labels: + - __name__ + regex: 'container_fs_writes_merged_total' + action: drop + - source_labels: + - __name__ + regex: 'container_fs_writes_total' + action: drop + - source_labels: + - __name__ + regex: 'container_last_seen' + action: drop + - source_labels: + - __name__ + regex: 'container_memory_cache' + action: drop + - source_labels: + - __name__ + regex: 'container_memory_failcnt' + action: drop + - source_labels: + - __name__ + regex: 'container_memory_max_usage_bytes' + action: drop + - source_labels: + - __name__ + regex: 'container_memory_rss' + action: drop + - source_labels: + - __name__ + regex: 'container_memory_swap' + action: drop + - source_labels: + - __name__ + regex: 'container_memory_usage_bytes' + action: drop + - source_labels: + - __name__ + regex: 'container_network_receive_errors_total' + action: drop + - source_labels: + - __name__ + regex: 'container_network_receive_packets_dropped_total' + action: drop + - source_labels: + - __name__ + regex: 'container_network_receive_packets_total' + action: drop + - source_labels: + - __name__ + regex: 'container_network_transmit_errors_total' + action: drop + - source_labels: + - __name__ + regex: 'container_network_transmit_packets_dropped_total' + action: drop + - source_labels: + - __name__ + regex: 'container_network_transmit_packets_total' + action: drop + - source_labels: + - __name__ + regex: 'container_spec_cpu_period' + action: drop + - source_labels: + - __name__ + regex: 'container_spec_cpu_shares' + action: drop + - source_labels: + - __name__ + regex: 'container_spec_memory_limit_bytes' + action: drop + - source_labels: + - __name__ + regex: 'container_spec_memory_reservation_limit_bytes' + action: drop + - source_labels: + - __name__ + regex: 'container_spec_memory_swap_limit_bytes' + action: drop + - source_labels: + - __name__ + regex: 'container_start_time_seconds' + action: drop + # Scrape config for API servers. + # + # Kubernetes exposes API servers as endpoints to the default/kubernetes + # service so this uses `endpoints` role and uses relabelling to only keep + # the endpoints associated with the default/kubernetes service using the + # default named port `https`. This works for single API server deployments as + # well as HA API server deployments. + - job_name: 'apiserver' + kubernetes_sd_configs: + - role: endpoints + scrape_interval: 45s + # Default to scraping over https. If required, just disable this or change to + # `http`. + scheme: https + # This TLS & bearer token file config is used to connect to the actual scrape + # endpoints for cluster components. This is separate to discovery auth + # configuration because discovery & scraping are two separate concerns in + # Prometheus. The discovery auth config is automatic if Prometheus runs inside + # the cluster. Otherwise, more config options have to be provided within the + # . + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + # If your node certificates are self-signed or use a different CA to the + # master CA, then disable certificate verification below. Note that + # certificate verification is an integral part of a secure infrastructure + # so this should only be disabled in a controlled environment. You can + # disable certificate verification by uncommenting the line below. + # + # insecure_skip_verify: true + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + # Keep only the default/kubernetes service endpoints for the https port. This + # will add targets for each API server which Kubernetes adds an endpoint to + # the default/kubernetes service. + relabel_configs: + - source_labels: + - __meta_kubernetes_namespace + - __meta_kubernetes_service_name + - __meta_kubernetes_endpoint_port_name + action: keep + regex: default;kubernetes;https + metric_relabel_configs: + - source_labels: + - __name__ + regex: 'apiserver_admission_controller_admission_latencies_seconds_bucket' + action: drop + - source_labels: + - __name__ + regex: 'rest_client_request_latency_seconds_bucket' + action: drop + - source_labels: + - __name__ + regex: 'apiserver_response_sizes_bucket' + action: drop + - source_labels: + - __name__ + regex: 'apiserver_admission_step_admission_latencies_seconds_bucket' + action: drop + - source_labels: + - __name__ + regex: 'apiserver_admission_controller_admission_latencies_seconds_count' + action: drop + - source_labels: + - __name__ + regex: 'apiserver_admission_controller_admission_latencies_seconds_sum' + action: drop + - source_labels: + - __name__ + regex: 'apiserver_request_latencies_summary' + action: drop + # Scrape config for service endpoints. + # + # The relabeling allows the actual service scrape endpoint to be configured + # via the following annotations: + # + # * `prometheus.io/scrape`: Only scrape services that have a value of `true` + # * `prometheus.io/scheme`: If the metrics endpoint is secured then you will need + # to set this to `https` & most likely set the `tls_config` of the scrape config. + # * `prometheus.io/path`: If the metrics path is not `/metrics` override this. + # * `prometheus.io/port`: If the metrics are exposed on a different port to the + # service then set this appropriately. + - job_name: 'openstack-exporter' + kubernetes_sd_configs: + - role: endpoints + scrape_interval: 60s + relabel_configs: + - source_labels: + - __meta_kubernetes_service_name + action: keep + regex: "openstack-metrics" + - source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scrape + action: keep + regex: true + - source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scheme + action: replace + target_label: __scheme__ + regex: (https?) + - source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_path + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: + - __address__ + - __meta_kubernetes_service_annotation_prometheus_io_port + action: replace + target_label: __address__ + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - source_labels: + - __meta_kubernetes_namespace + action: replace + target_label: kubernetes_namespace + - source_labels: + - __meta_kubernetes_service_name + action: replace + target_label: instance + - source_labels: + - __meta_kubernetes_service_name + action: replace + target_label: kubernetes_name + - source_labels: + - __meta_kubernetes_service_name + target_label: job + replacement: ${1} + - job_name: 'node-exporter' + kubernetes_sd_configs: + - role: endpoints + scrape_interval: 60s + relabel_configs: + - source_labels: + - __meta_kubernetes_service_name + action: keep + regex: 'node-exporter' + - source_labels: + - __meta_kubernetes_pod_node_name + action: replace + target_label: hostname + - job_name: 'kubernetes-service-endpoints' + kubernetes_sd_configs: + - role: endpoints + scrape_interval: 60s + relabel_configs: + - source_labels: + - __meta_kubernetes_service_name + action: drop + regex: '(openstack-metrics|prom-metrics|ceph-mgr|node-exporter)' + - source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scrape + action: keep + regex: true + - source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_scheme + action: replace + target_label: __scheme__ + regex: (https?) + - source_labels: + - __meta_kubernetes_service_annotation_prometheus_io_path + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: + - __address__ + - __meta_kubernetes_service_annotation_prometheus_io_port + action: replace + target_label: __address__ + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - source_labels: + - __meta_kubernetes_namespace + action: replace + target_label: kubernetes_namespace + - source_labels: + - __meta_kubernetes_service_name + action: replace + target_label: kubernetes_name + - source_labels: + - __meta_kubernetes_service_name + target_label: job + replacement: ${1} + # Example scrape config for pods + # + # The relabeling allows the actual pod scrape endpoint to be configured via the + # following annotations: + # + # * `prometheus.io/scrape`: Only scrape pods that have a value of `true` + # * `prometheus.io/path`: If the metrics path is not `/metrics` override this. + # * `prometheus.io/port`: Scrape the pod on the indicated port instead of the + # pod's declared ports (default is a port-free target if none are declared). + - job_name: 'kubernetes-pods' + kubernetes_sd_configs: + - role: pod + relabel_configs: + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: kubernetes_pod_name + - job_name: calico-etcd + kubernetes_sd_configs: + - role: service + scrape_interval: 20s + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - action: keep + source_labels: + - __meta_kubernetes_service_name + regex: "calico-etcd" + - action: keep + source_labels: + - __meta_kubernetes_namespace + regex: kube-system + target_label: namespace + - source_labels: + - __meta_kubernetes_pod_name + target_label: pod + - source_labels: + - __meta_kubernetes_service_name + target_label: service + - source_labels: + - __meta_kubernetes_service_name + target_label: job + replacement: ${1} + - source_labels: + - __meta_kubernetes_service_label + target_label: job + regex: calico-etcd + replacement: ${1} + - target_label: endpoint + replacement: "calico-etcd" + - job_name: ceph-mgr + kubernetes_sd_configs: + - role: service + scrape_interval: 20s + relabel_configs: + - action: labelmap + regex: __meta_kubernetes_service_label_(.+) + - action: keep + source_labels: + - __meta_kubernetes_service_name + regex: "ceph-mgr" + - source_labels: + - __meta_kubernetes_service_port_name + action: drop + regex: 'ceph-mgr' + - action: keep + source_labels: + - __meta_kubernetes_namespace + regex: ceph + target_label: namespace + - source_labels: + - __meta_kubernetes_pod_name + target_label: pod + - source_labels: + - __meta_kubernetes_service_name + target_label: service + - source_labels: + - __meta_kubernetes_service_name + target_label: job + replacement: ${1} + - source_labels: + - __meta_kubernetes_service_label + target_label: job + regex: ceph-mgr + replacement: ${1} + - target_label: endpoint + replacement: "ceph-mgr" + alerting: + alertmanagers: + - kubernetes_sd_configs: + - role: pod + tls_config: + ca_file: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + relabel_configs: + - source_labels: [__meta_kubernetes_pod_label_application] + regex: prometheus-alertmanager + action: keep + - source_labels: [__meta_kubernetes_pod_container_port_name] + regex: alerts-api + action: keep + - source_labels: [__meta_kubernetes_pod_container_port_name] + regex: peer-mesh + action: drop + rules: [] +... diff --git a/rabbitmq/.helmignore b/rabbitmq/.helmignore new file mode 100644 index 0000000000..3ca0aad82e --- /dev/null +++ b/rabbitmq/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj + + diff --git a/rabbitmq/Chart.yaml b/rabbitmq/Chart.yaml new file mode 100644 index 0000000000..b99813e8b7 --- /dev/null +++ b/rabbitmq/Chart.yaml @@ -0,0 +1,24 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: v3.12.0 +description: OpenStack-Helm RabbitMQ +name: rabbitmq +version: 2024.2.0 +home: https://github.com/rabbitmq/rabbitmq-server +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit + version: ">= 0.1.0" +... diff --git a/rabbitmq/templates/bin/_rabbitmq-cookie.sh.tpl b/rabbitmq/templates/bin/_rabbitmq-cookie.sh.tpl new file mode 100644 index 0000000000..911ae4f6f5 --- /dev/null +++ b/rabbitmq/templates/bin/_rabbitmq-cookie.sh.tpl @@ -0,0 +1,21 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +cp -vf /run/lib/rabbitmq/.erlang.cookie /var/lib/rabbitmq/.erlang.cookie +chown "rabbitmq" /var/lib/rabbitmq/.erlang.cookie +chmod 0600 /var/lib/rabbitmq/.erlang.cookie diff --git a/rabbitmq/templates/bin/_rabbitmq-liveness.sh.tpl b/rabbitmq/templates/bin/_rabbitmq-liveness.sh.tpl new file mode 100644 index 0000000000..62cb3da6ad --- /dev/null +++ b/rabbitmq/templates/bin/_rabbitmq-liveness.sh.tpl @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -e + +if [ -f /tmp/rabbit-disable-liveness-probe ]; then + exit 0 +else + exec rabbitmqctl ping +fi diff --git a/rabbitmq/templates/bin/_rabbitmq-password-hash.py.tpl b/rabbitmq/templates/bin/_rabbitmq-password-hash.py.tpl new file mode 100644 index 0000000000..79f9b76fb1 --- /dev/null +++ b/rabbitmq/templates/bin/_rabbitmq-password-hash.py.tpl @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +See here for explanation: +http://lists.rabbitmq.com/pipermail/rabbitmq-discuss/2011-May/012765.html +*/}} + +from __future__ import print_function +import base64 +import json +import os +import hashlib +import re + +user = os.environ['RABBITMQ_ADMIN_USERNAME'] +password = os.environ['RABBITMQ_ADMIN_PASSWORD'] +guest_password = os.environ['RABBITMQ_GUEST_PASSWORD'] +output_file = os.environ['RABBITMQ_DEFINITION_FILE'] + +def hash_rabbit_password(password): + salt = os.urandom(4) + tmp0 = salt + password.encode('utf-8') + tmp1 = hashlib.sha512(tmp0).digest() + salted_hash = salt + tmp1 + pass_hash = base64.b64encode(salted_hash) + return pass_hash.decode("utf-8") + +output = { + "users": [{ + "name": user, + "password_hash": hash_rabbit_password(password), + "hashing_algorithm": "rabbit_password_hashing_sha512", + "tags": "administrator" + }, + { + "name": "guest", + "password_hash": hash_rabbit_password(guest_password), + "hashing_algorithm": "rabbit_password_hashing_sha512", + "tags": "administrator" + } + ] +} + +if 'RABBITMQ_USERS' in os.environ: + output.update({'vhosts': []}) + output.update({'permissions': []}) + users_creds = json.loads(os.environ['RABBITMQ_USERS']) + for user, creds in users_creds.items(): + if 'auth' in creds: + for auth_key, auth_val in creds['auth'].items(): + username = auth_val['username'] + password = auth_val['password'] + user_struct = { + "name": username, + "password_hash": hash_rabbit_password(password), + "hashing_algorithm": "rabbit_password_hashing_sha512", + "tags": "" + } + output['users'].append(user_struct) + if 'path' in creds: + for path in ( + creds["path"] + if isinstance(creds["path"], list) + else [creds["path"]] + ): + vhost = re.sub("^/", "", path) + vhost_struct = {"name": vhost} + + perm_struct = { + "user": username, + "vhost": vhost, + "configure": ".*", + "write": ".*", + "read": ".*" + } + + output['vhosts'].append(vhost_struct) + output['permissions'].append(perm_struct) + +if 'RABBITMQ_AUXILIARY_CONFIGURATION' in os.environ: + aux_conf = json.loads(os.environ['RABBITMQ_AUXILIARY_CONFIGURATION']) + if aux_conf.get('policies', []): + output['policies'] = aux_conf['policies'] + if aux_conf.get('bindings', []): + output['bindings'] = aux_conf['bindings'] + if aux_conf.get('queues', []): + output['queues'] = aux_conf['queues'] + if aux_conf.get('exchanges', []): + output['exchanges'] = aux_conf['exchanges'] + +with open(output_file, 'w') as f: + f.write(json.dumps(output)) + f.close() diff --git a/rabbitmq/templates/bin/_rabbitmq-readiness.sh.tpl b/rabbitmq/templates/bin/_rabbitmq-readiness.sh.tpl new file mode 100644 index 0000000000..bf49d9eff2 --- /dev/null +++ b/rabbitmq/templates/bin/_rabbitmq-readiness.sh.tpl @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -e + +if [ -f /tmp/rabbit-disable-readiness ]; then + exit 1 +else + exec rabbitmqctl ping +fi diff --git a/rabbitmq/templates/bin/_rabbitmq-start.sh.tpl b/rabbitmq/templates/bin/_rabbitmq-start.sh.tpl new file mode 100644 index 0000000000..4ef849fd10 --- /dev/null +++ b/rabbitmq/templates/bin/_rabbitmq-start.sh.tpl @@ -0,0 +1,100 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +function check_if_open () { + HOST=$1 + PORT=$2 + timeout 10 bash -c "true &>/dev/null /dev/null +} + +get_node_name () { + TARGET_POD=$1 + POD_NAME_PREFIX="$(echo "${MY_POD_NAME}" | awk 'BEGIN{FS=OFS="-"}{NF--; print}')" + echo "${RABBITMQ_NODENAME}" | awk -F "@${MY_POD_NAME}." "{ print \$1 \"@${POD_NAME_PREFIX}-${TARGET_POD}.\" \$2 }" +} + +function check_rabbit_node_ready () { + TARGET_POD=$1 + CLUSTER_SEED_NAME="$(get_node_name ${TARGET_POD})" + CLUSTER_SEED_HOST="$(echo "${CLUSTER_SEED_NAME}" | awk -F '@' '{ print $NF }')" + check_rabbit_node_health "${CLUSTER_SEED_NAME}" && \ + check_if_open "${CLUSTER_SEED_HOST}" "${PORT_HTTP}" && \ + check_if_open "${CLUSTER_SEED_HOST}" "${PORT_AMPQ}" && \ + check_if_open "${CLUSTER_SEED_HOST}" "${PORT_CLUSTERING}" +} + +POD_INCREMENT=$(echo "${MY_POD_NAME}" | awk -F '-' '{print $NF}') +if ! [ "${POD_INCREMENT}" -eq "0" ] && ! [ -d "/var/lib/rabbitmq/mnesia" ] ; then + echo 'This is not the 1st rabbit pod & has not been initialised' + # disable liveness probe as it may take some time for the pod to come online. + touch /tmp/rabbit-disable-liveness-probe + POD_NAME_PREFIX="$(echo "${MY_POD_NAME}" | awk 'BEGIN{FS=OFS="-"}{NF--; print}')" + for TARGET_POD in $(seq 0 +1 $((POD_INCREMENT - 1 ))); do + END=$(($(date +%s) + 900)) + while ! check_rabbit_node_ready "${TARGET_POD}"; do + sleep 5 + if [ "$(date +%s)" -gt "$END" ]; then + echo "RabbitMQ pod ${TARGET_POD} not ready in time" + exit 1 + fi + done + done + + function reset_rabbit () { + rabbitmqctl shutdown || true + find /var/lib/rabbitmq/* ! -name 'definitions.json' ! -name '.erlang.cookie' -exec rm -rf {} + + exit 1 + } + + # Start RabbitMQ, but disable readiness from being reported so the pod is not + # marked as up prematurely. + touch /tmp/rabbit-disable-readiness + rabbitmq-server & + + # Wait for server to start, and reset if it does not + END=$(($(date +%s) + 180)) + while ! rabbitmqctl -q cluster_status; do + sleep 5 + NOW=$(date +%s) + [ $NOW -gt $END ] && reset_rabbit + done + + # Wait for server to join cluster, reset if it does not + POD_INCREMENT=$(echo "${MY_POD_NAME}" | awk -F '-' '{print $NF}') + END=$(($(date +%s) + 180)) + while ! rabbitmqctl -l --node $(get_node_name 0) -q cluster_status | grep -q "$(get_node_name ${POD_INCREMENT})"; do + sleep 5 + NOW=$(date +%s) + [ $NOW -gt $END ] && reset_rabbit + done + + # Shutdown the inital server + rabbitmqctl shutdown + + rm -fv /tmp/rabbit-disable-readiness /tmp/rabbit-disable-liveness-probe +fi + +{{- if .Values.forceBoot.enabled }} +if [ "${POD_INCREMENT}" -eq "0" ] && [ -d "/var/lib/rabbitmq/mnesia/${RABBITMQ_NODENAME}" ]; then rabbitmqctl force_boot; fi +{{- end}} +exec rabbitmq-server diff --git a/rabbitmq/templates/bin/_rabbitmq-test.sh.tpl b/rabbitmq/templates/bin/_rabbitmq-test.sh.tpl new file mode 100644 index 0000000000..46abf3ec96 --- /dev/null +++ b/rabbitmq/templates/bin/_rabbitmq-test.sh.tpl @@ -0,0 +1,97 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +# Extract connection details +RABBIT_HOSTNAME=`echo $RABBITMQ_ADMIN_CONNECTION | awk -F'[@]' '{print $2}' \ + | awk -F'[:/]' '{print $1}'` +RABBIT_PORT=`echo $RABBITMQ_ADMIN_CONNECTION | awk -F'[@]' '{print $2}' \ + | awk -F'[:/]' '{print $2}'` + +set +x +# Extract Admin User creadential +RABBITMQ_ADMIN_USERNAME=`echo $RABBITMQ_ADMIN_CONNECTION | awk -F'[@]' '{print $1}' \ + | awk -F'[//:]' '{print $4}'` +RABBITMQ_ADMIN_PASSWORD=`echo $RABBITMQ_ADMIN_CONNECTION | awk -F'[@]' '{print $1}' \ + | awk -F'[//:]' '{print $5}'` +set -x + +function rabbitmqadmin_authed () { + set +x + rabbitmqadmin \ +{{- if .Values.manifests.certificates }} + --ssl \ + --ssl-disable-hostname-verification \ + --ssl-ca-cert-file="/etc/rabbitmq/certs/ca.crt" \ + --ssl-cert-file="/etc/rabbitmq/certs/tls.crt" \ + --ssl-key-file="/etc/rabbitmq/certs/tls.key" \ +{{- end }} + --host="${RABBIT_HOSTNAME}" \ + --port="${RABBIT_PORT}" \ + --username="${RABBITMQ_ADMIN_USERNAME}" \ + --password="${RABBITMQ_ADMIN_PASSWORD}" \ + ${@} + set -x +} + +function rabbit_check_node_count () { + echo "Checking node count " + NODES_IN_CLUSTER=$(rabbitmqadmin_authed list nodes -f bash | wc -w) + if [ "$NODES_IN_CLUSTER" -eq "$RABBIT_REPLICA_COUNT" ]; then + echo "Number of nodes in cluster ($NODES_IN_CLUSTER) match number of desired pods ($NODES_IN_CLUSTER)" + else + echo "Number of nodes in cluster ($NODES_IN_CLUSTER) does not match number of desired pods ($RABBIT_REPLICA_COUNT)" + exit 1 + fi +} +# Check node count +rabbit_check_node_count + +function rabbit_find_partitions () { + NODE_INFO=$(mktemp) + rabbitmqadmin_authed list nodes -f pretty_json | tee "${NODE_INFO}" + cat "${NODE_INFO}" | python3 -c " +import json, sys, traceback +print('Checking cluster partitions') +obj=json.load(sys.stdin) +for num, node in enumerate(obj): + try: + partition = node['partitions'] + if partition: + raise Exception('cluster partition found: %s' % partition) + except KeyError: + print('Error: partition key not found for node %s' % node) +print('No cluster partitions found') + " + rm -vf "${NODE_INFO}" +} +rabbit_find_partitions + +function rabbit_check_users_match () { + echo "Checking users match on all nodes" + NODES=$(rabbitmqadmin_authed list nodes -f bash) + USER_LIST=$(mktemp --directory) + echo "Found the following nodes: ${NODES}" + for NODE in ${NODES}; do + echo "Checking Node: ${NODE#*@}" + rabbitmqadmin_authed list users -f bash > ${USER_LIST}/${NODE#*@} + done + cd ${USER_LIST}; diff -q --from-file $(ls ${USER_LIST}) + echo "User lists match for all nodes" +} +# Check users match on all nodes +rabbit_check_users_match diff --git a/rabbitmq/templates/bin/_rabbitmq-wait-for-cluster.sh.tpl b/rabbitmq/templates/bin/_rabbitmq-wait-for-cluster.sh.tpl new file mode 100644 index 0000000000..215e5b9050 --- /dev/null +++ b/rabbitmq/templates/bin/_rabbitmq-wait-for-cluster.sh.tpl @@ -0,0 +1,80 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -e + +# Extract connection details +RABBIT_HOSTNAME=`echo $RABBITMQ_ADMIN_CONNECTION | awk -F'[@]' '{print $2}' \ + | awk -F'[:/]' '{print $1}'` +RABBIT_PORT=`echo $RABBITMQ_ADMIN_CONNECTION | awk -F'[@]' '{print $2}' \ + | awk -F'[:/]' '{print $2}'` + +# Extract Admin User creadential +RABBITMQ_ADMIN_USERNAME=`echo $RABBITMQ_ADMIN_CONNECTION | awk -F'[@]' '{print $1}' \ + | awk -F'[//:]' '{print $4}'` +RABBITMQ_ADMIN_PASSWORD=`echo $RABBITMQ_ADMIN_CONNECTION | awk -F'[@]' '{print $1}' \ + | awk -F'[//:]' '{print $5}'` + +set -ex + +function rabbitmqadmin_authed () { + set +x + rabbitmqadmin \ +{{- if .Values.manifests.certificates }} + --ssl \ + --ssl-disable-hostname-verification \ + --ssl-ca-cert-file="/etc/rabbitmq/certs/ca.crt" \ + --ssl-cert-file="/etc/rabbitmq/certs/tls.crt" \ + --ssl-key-file="/etc/rabbitmq/certs/tls.key" \ +{{- end }} + --host="${RABBIT_HOSTNAME}" \ + --port="${RABBIT_PORT}" \ + --username="${RABBITMQ_ADMIN_USERNAME}" \ + --password="${RABBITMQ_ADMIN_PASSWORD}" \ + ${@} + set -x +} + +function active_rabbit_nodes () { + rabbitmqadmin_authed list nodes -f bash | wc -w +} + +until test "$(active_rabbit_nodes)" -ge "$RABBIT_REPLICA_COUNT"; do + echo "Waiting for number of nodes in cluster to meet or exceed number of desired pods ($RABBIT_REPLICA_COUNT)" + sleep 10 +done + +function sorted_node_list () { + rabbitmqadmin_authed list nodes -f bash | tr ' ' '\n' | sort | tr '\n' ' ' +} + +if test "$(active_rabbit_nodes)" -gt "$RABBIT_REPLICA_COUNT"; then + echo "There are more nodes registed in the cluster than desired, pruning the cluster" + PRIMARY_NODE="$(sorted_node_list | awk '{ print $1; exit }')" + until rabbitmqctl -l -n "${PRIMARY_NODE}" cluster_status >/dev/null 2>&1 ; do + echo "Waiting for primary node to return cluster status" + sleep 10 + done + echo "Current cluster:" + rabbitmqctl -l -n "${PRIMARY_NODE}" cluster_status + NODES_TO_REMOVE="$(sorted_node_list | awk "{print substr(\$0, index(\$0,\$$((RABBIT_REPLICA_COUNT+1))))}")" + for NODE in ${NODES_TO_REMOVE}; do + rabbitmqctl -l -n "${NODE}" stop_app || true + rabbitmqctl -l -n "${PRIMARY_NODE}" forget_cluster_node "${NODE}" + done + echo "Updated cluster:" + rabbitmqctl -l -n "${PRIMARY_NODE}" cluster_status +fi diff --git a/rabbitmq/templates/certificates.yaml b/rabbitmq/templates/certificates.yaml new file mode 100644 index 0000000000..d7f88e5882 --- /dev/null +++ b/rabbitmq/templates/certificates.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.certificates -}} +{{ dict "envAll" . "service" "oslo_messaging" "type" "internal" | include "helm-toolkit.manifests.certificates" }} +{{- end -}} diff --git a/rabbitmq/templates/configmap-bin.yaml b/rabbitmq/templates/configmap-bin.yaml new file mode 100644 index 0000000000..d2ffbb9f1a --- /dev/null +++ b/rabbitmq/templates/configmap-bin.yaml @@ -0,0 +1,47 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if (.Values.global).subchart_release_name }} +{{- $_ := set . "deployment_name" .Chart.Name }} +{{- else }} +{{- $_ := set . "deployment_name" .Release.Name }} +{{- end }} + +{{- if .Values.manifests.configmap_bin }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ printf "%s-%s" .deployment_name "rabbitmq-bin" | quote }} +data: +{{- if .Values.images.local_registry.active }} + image-repo-sync.sh: | +{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} +{{- end }} + rabbitmq-test.sh: | +{{ tuple "bin/_rabbitmq-test.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + rabbitmq-liveness.sh: | +{{ tuple "bin/_rabbitmq-liveness.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + rabbitmq-readiness.sh: | +{{ tuple "bin/_rabbitmq-readiness.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + rabbitmq-start.sh: | +{{ tuple "bin/_rabbitmq-start.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + rabbitmq-cookie.sh: | +{{ tuple "bin/_rabbitmq-cookie.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + rabbitmq-password-hash.py: | +{{ tuple "bin/_rabbitmq-password-hash.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + rabbitmq-wait-for-cluster.sh: | +{{ tuple "bin/_rabbitmq-wait-for-cluster.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{ end }} diff --git a/rabbitmq/templates/configmap-etc.yaml b/rabbitmq/templates/configmap-etc.yaml new file mode 100644 index 0000000000..7544a1c04e --- /dev/null +++ b/rabbitmq/templates/configmap-etc.yaml @@ -0,0 +1,94 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if (.Values.global).subchart_release_name }} +{{- $_ := set . "deployment_name" .Chart.Name }} +{{- else }} +{{- $_ := set . "deployment_name" .Release.Name }} +{{- end }} + +{{/* +(aostapenko) rounds cpu limit in any permissible format to integer value (min 1) +"100m" -> 1 +"1100m" -> 1 +"10900m" -> 10 +0.3 -> 1 +5.4 -> 5 +*/}} +{{- define "get_erlvm_scheduler_num" -}} +{{- $val := . | toString -}} +{{- if regexMatch "^[0-9]*m$" $val -}} +{{- $val = div (float64 (trimSuffix "m" $val)) 1000 -}} +{{- end -}} +{{/* NOTE(aostapenko) String with floating number does not convert well to int*/}} +{{- $val | float64 | int | default 1 -}} +{{- end -}} + +{{- if .Values.manifests.configmap_etc }} +{{- $envAll := . }} + +{{- if empty $envAll.Values.conf.rabbitmq.cluster_formation.k8s.host -}} +{{- $_ := print "kubernetes.default.svc." $envAll.Values.endpoints.cluster_domain_suffix | set $envAll.Values.conf.rabbitmq.cluster_formation.k8s "host" -}} +{{- end -}} + +{{- if .Values.manifests.certificates }} +{{- $_ := print "none" | set $envAll.Values.conf.rabbitmq.listeners "tcp" -}} +{{- $_ := tuple "oslo_messaging" "internal" "amqp" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | set $envAll.Values.conf.rabbitmq.listeners "ssl.1" -}} +{{- $_ := tuple "oslo_messaging" "internal" "https" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | set $envAll.Values.conf.rabbitmq "management.ssl.port" -}} +{{- else }} +{{- $_ := print $envAll.Values.conf.bind_address ":" ( tuple "oslo_messaging" "internal" "amqp" . | include "helm-toolkit.endpoints.endpoint_port_lookup") | set $envAll.Values.conf.rabbitmq.listeners.tcp "1" -}} +{{- $_ := tuple "oslo_messaging" "internal" "http" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | set $envAll.Values.conf.rabbit_additonal_conf "management.listener.port" -}} +{{- end }} + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ printf "%s-%s" $envAll.deployment_name "rabbitmq-etc" | quote }} +data: + enabled_plugins: | +{{ tuple "etc/_enabled_plugins.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + rabbitmq.conf: | +{{ include "rabbitmq.utils.to_rabbit_config" $envAll.Values.conf.rabbitmq | indent 4 }} +{{- if not .Values.manifests.certificates }} +{{ include "rabbitmq.utils.to_rabbit_config" $envAll.Values.conf.rabbit_additonal_conf | indent 4 }} +{{- end }} + +{{- if .Values.conf.rabbit_advanced_config.enabled }} + advanced.config: | + [ + {rabbit, [ + {default_consumer_prefetch, {false,{{ .Values.conf.rabbit_advanced_config.default_consumer_prefetch }}}} + ] + } + ]. +{{- end }} + +{{- $erlvm_scheduler_num := include "get_erlvm_scheduler_num" .Values.pod.resources.server.limits.cpu }} +{{- $erlvm_scheduler_conf := printf "+S %s:%s" $erlvm_scheduler_num $erlvm_scheduler_num }} +{{- if .Values.manifests.config_ipv6 }} + rabbitmq-env.conf: | + SERVER_ADDITIONAL_ERL_ARGS={{ printf "+A 128 -kernel inetrc '/etc/rabbitmq/erl_inetrc' -proto_dist inet6_tcp %s" $erlvm_scheduler_conf | quote }} + CTL_ERL_ARGS="-proto_dist inet6_tcp" + erl_inetrc: | + {inet6, true}. +{{- else }} + rabbitmq-env.conf: | + SERVER_ADDITIONAL_ERL_ARGS={{ $erlvm_scheduler_conf | quote }} +{{- end }} +{{ if not .Values.conf.prometheus_exporter.rabbitmq_mgmt_metrics_collector_disabled }} + management_agent.disable_metrics_collector.conf: | + management_agent.disable_metrics_collector = false +{{- end }} +{{ end }} diff --git a/rabbitmq/templates/etc/_enabled_plugins.tpl b/rabbitmq/templates/etc/_enabled_plugins.tpl new file mode 100644 index 0000000000..4c2d8cc49a --- /dev/null +++ b/rabbitmq/templates/etc/_enabled_plugins.tpl @@ -0,0 +1,15 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +[{{ include "helm-toolkit.utils.joinListWithComma" .Values.conf.enabled_plugins }}]. diff --git a/rabbitmq/templates/ingress-management.yaml b/rabbitmq/templates/ingress-management.yaml new file mode 100644 index 0000000000..25be361422 --- /dev/null +++ b/rabbitmq/templates/ingress-management.yaml @@ -0,0 +1,29 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if (.Values.global).subchart_release_name }} +{{- $_ := set . "deployment_name" .Chart.Name }} +{{- else }} +{{- $_ := set . "deployment_name" .Release.Name }} +{{- end }} + +{{- if and .Values.manifests.ingress_management .Values.network.management.ingress.public }} +{{- $envAll := . }} +{{- if empty $envAll.Values.endpoints.oslo_messaging.hosts.public }} +{{- $service_public_name := .deployment_name | trunc 12 }} +{{- $_ := set $envAll.Values.endpoints.oslo_messaging.hosts "public" ( printf "%s-%s-%s" $service_public_name "mgr" ( $service_public_name | sha256sum | trunc 6 )) }} +{{- end }} +{{- $ingressOpts := dict "envAll" . "backendService" "management" "backendServiceType" "oslo_messaging" "backendPort" "http" -}} +{{ $ingressOpts | include "helm-toolkit.manifests.ingress" }} +{{- end }} diff --git a/rabbitmq/templates/job-cluster-wait.yaml b/rabbitmq/templates/job-cluster-wait.yaml new file mode 100644 index 0000000000..1c4378c708 --- /dev/null +++ b/rabbitmq/templates/job-cluster-wait.yaml @@ -0,0 +1,119 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if (.Values.global).subchart_release_name }} +{{- $_ := set . "deployment_name" .Chart.Name }} +{{- else }} +{{- $_ := set . "deployment_name" .Release.Name }} +{{- end }} + +{{- if .Values.manifests.job_cluster_wait }} +{{- $envAll := . }} + +{{- $serviceAccountName := print .deployment_name "-cluster-wait" }} +{{ tuple $envAll "cluster_wait" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} + +{{- $protocol := "http" }} +{{- if $envAll.Values.manifests.certificates }} +{{- $protocol = "https" }} +{{- end }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: "{{.deployment_name}}-cluster-wait" + labels: +{{ tuple $envAll "rabbitmq" "cluster-wait" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +{{- if .Values.helm3_hook }} + "helm.sh/hook": "post-install,post-upgrade" + "helm.sh/hook-weight": "5" + "helm.sh/hook-delete-policy": before-hook-creation +{{- end }} +spec: + template: + metadata: + labels: +{{ tuple $envAll "rabbitmq" "cluster-wait" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} +{{ dict "envAll" $envAll "podName" "rabbitmq-cluster-wait" "containerNames" (list "init" "rabbitmq-cookie" "rabbitmq-rabbitmq-cluster-wait" ) | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "cluster_wait" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure +{{ if $envAll.Values.pod.tolerations.rabbitmq.enabled }} +{{ tuple $envAll "rabbitmq" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }} +{{ end }} + nodeSelector: + {{ $envAll.Values.labels.jobs.node_selector_key }}: {{ $envAll.Values.labels.test.node_selector_value | quote }} + initContainers: +{{ tuple $envAll "cluster_wait" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + - name: rabbitmq-cookie +{{ tuple $envAll "scripted_test" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ dict "envAll" $envAll "application" "cluster_wait" "container" "rabbitmq_cookie" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/rabbitmq-cookie.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: rabbitmq-bin + mountPath: /tmp/rabbitmq-cookie.sh + subPath: rabbitmq-cookie.sh + readOnly: true + - name: rabbitmq-data + mountPath: /var/lib/rabbitmq + - name: rabbitmq-erlang-cookie + mountPath: /var/run/lib/rabbitmq/.erlang.cookie + subPath: erlang_cookie + readOnly: true + containers: + - name: rabbitmq-rabbitmq-cluster-wait +{{ tuple $envAll "scripted_test" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ dict "envAll" $envAll "application" "cluster_wait" "container" "rabbitmq_cluster_wait" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + env: + - name: RABBITMQ_ADMIN_CONNECTION + value: {{ tuple "oslo_messaging" "internal" "user" $protocol $envAll | include "helm-toolkit.endpoints.authenticated_endpoint_uri_lookup" | quote }} + - name: RABBIT_REPLICA_COUNT + value: {{ $envAll.Values.pod.replicas.server | quote }} + command: + - /tmp/rabbitmq-wait-for-cluster.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: rabbitmq-bin + mountPath: /tmp/rabbitmq-wait-for-cluster.sh + subPath: rabbitmq-wait-for-cluster.sh + readOnly: true + - name: rabbitmq-data + mountPath: /var/lib/rabbitmq +{{ dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.secrets.tls.oslo_messaging.server.internal "path" "/etc/rabbitmq/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: rabbitmq-data + emptyDir: {} + - name: rabbitmq-bin + configMap: + name: {{ printf "%s-%s" $envAll.deployment_name "rabbitmq-bin" | quote }} + defaultMode: 0555 + - name: rabbitmq-erlang-cookie + secret: + secretName: {{ printf "%s-%s" $envAll.deployment_name "erlang-cookie" | quote }} + defaultMode: 0444 +{{ dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.secrets.tls.oslo_messaging.server.internal | include "helm-toolkit.snippets.tls_volume" | indent 8 }} +{{- end }} diff --git a/rabbitmq/templates/job-image-repo-sync.yaml b/rabbitmq/templates/job-image-repo-sync.yaml new file mode 100644 index 0000000000..8fd379f953 --- /dev/null +++ b/rabbitmq/templates/job-image-repo-sync.yaml @@ -0,0 +1,21 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "rabbitmq" -}} +{{- if .Values.pod.tolerations.rabbitmq.enabled -}} +{{- $_ := set $imageRepoSyncJob "tolerationsEnabled" true -}} +{{- end -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/rabbitmq/templates/monitoring/prometheus/exporter-deployment.yaml b/rabbitmq/templates/monitoring/prometheus/exporter-deployment.yaml new file mode 100644 index 0000000000..b08fc88571 --- /dev/null +++ b/rabbitmq/templates/monitoring/prometheus/exporter-deployment.yaml @@ -0,0 +1,119 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if (.Values.global).subchart_release_name }} +{{- $_ := set . "deployment_name" .Chart.Name }} +{{- else }} +{{- $_ := set . "deployment_name" .Release.Name }} +{{- end }} + +{{- define "exporterProbeTemplate" }} +httpGet: + scheme: HTTP + path: / + port: {{ tuple "prometheus_rabbitmq_exporter" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} +{{- end }} + +{{- if and .Values.manifests.monitoring.prometheus.deployment_exporter .Values.monitoring.prometheus.enabled }} +{{- $envAll := . }} + +{{- $rcControllerName := printf "%s-%s" $envAll.deployment_name "rabbitmq-exporter" }} +{{ tuple $envAll "prometheus_rabbitmq_exporter" $rcControllerName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} + +{{- $protocol := "http" }} +{{- if $envAll.Values.manifests.certificates }} +{{- $protocol = "https" }} +{{- end }} + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ $rcControllerName | quote }} + labels: +{{ tuple $envAll "prometheus_rabbitmq_exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + replicas: {{ $envAll.Values.pod.replicas.prometheus_rabbitmq_exporter }} + selector: + matchLabels: +{{ tuple $envAll "prometheus_rabbitmq_exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} +{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "prometheus_rabbitmq_exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + namespace: {{ $envAll.Values.endpoints.prometheus_rabbitmq_exporter.namespace }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} +{{ dict "envAll" $envAll "podName" "prometheus-rabbitmq-exporter" "containerNames" (list "init" "rabbitmq-exporter") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "exporter" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $rcControllerName | quote }} + nodeSelector: + {{ $envAll.Values.labels.prometheus_rabbitmq_exporter.node_selector_key }}: {{ $envAll.Values.labels.prometheus_rabbitmq_exporter.node_selector_value | quote }} + terminationGracePeriodSeconds: {{ $envAll.Values.pod.lifecycle.termination_grace_period.prometheus_rabbitmq_exporter.timeout | default "30" }} + initContainers: +{{ tuple $envAll "prometheus_rabbitmq_exporter" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: rabbitmq-exporter +{{ tuple $envAll "prometheus_rabbitmq_exporter" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.prometheus_rabbitmq_exporter | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "exporter" "container" "rabbitmq_exporter" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} +{{ dict "envAll" $envAll "component" "prometheus_rabbitmq_exporter" "container" "rabbitmq_exporter" "type" "readiness" "probeTemplate" (include "exporterProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | trim | indent 10 }} +{{ dict "envAll" $envAll "component" "prometheus_rabbitmq_exporter" "container" "rabbitmq_exporter" "type" "liveness" "probeTemplate" (include "exporterProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | trim | indent 10 }} + ports: + - name: metrics + containerPort: {{ tuple "prometheus_rabbitmq_exporter" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + env: + - name: RABBIT_TIMEOUT + value: "{{ .Values.conf.rabbitmq_exporter.rabbit_timeout }}" + - name: RABBIT_URL + value: {{ printf "%s" $protocol }}://{{ tuple "oslo_messaging" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }}:{{ tuple "oslo_messaging" "internal" $protocol . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + - name: RABBIT_USER + valueFrom: + secretKeyRef: + name: {{ printf "%s-%s" $envAll.deployment_name "admin-user" | quote }} + key: RABBITMQ_ADMIN_USERNAME + - name: RABBIT_PASSWORD + valueFrom: + secretKeyRef: + name: {{ printf "%s-%s" $envAll.deployment_name "admin-user" | quote }} + key: RABBITMQ_ADMIN_PASSWORD + - name: RABBIT_CAPABILITIES + value: {{ include "helm-toolkit.utils.joinListWithComma" $envAll.Values.conf.prometheus_exporter.capabilities | quote }} + - name: PUBLISH_PORT + value: {{ tuple "prometheus_rabbitmq_exporter" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" | quote }} + - name: LOG_LEVEL + value: {{ $envAll.Values.conf.prometheus_exporter.log_level | quote }} + - name: SKIPVERIFY + value: {{ $envAll.Values.conf.prometheus_exporter.skipverify | quote }} + - name: SKIP_QUEUES + value: {{ $envAll.Values.conf.prometheus_exporter.skip_queues | default "^$" | quote }} + - name: INCLUDE_QUEUES + value: {{ $envAll.Values.conf.prometheus_exporter.include_queues | default ".*" | quote }} + - name: RABBIT_EXPORTERS + value: {{ $envAll.Values.conf.prometheus_exporter.rabbit_exporters | default "overview,exchange,node,queue" | quote }} +{{- if $envAll.Values.manifests.certificates }} + - name: CAFILE + value: "/etc/rabbitmq/certs/ca.crt" + - name: CERTFILE + value: "/etc/rabbitmq/certs/tls.crt" + - name: KEYFILE + value: "/etc/rabbitmq/certs/tls.key" + volumeMounts: +{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.oslo_messaging.server.internal "path" "/etc/rabbitmq/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }} + volumes: +{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.oslo_messaging.server.internal | include "helm-toolkit.snippets.tls_volume" | indent 8 }} +{{- end }} +{{- end }} diff --git a/rabbitmq/templates/monitoring/prometheus/exporter-network-policy.yaml b/rabbitmq/templates/monitoring/prometheus/exporter-network-policy.yaml new file mode 100644 index 0000000000..504572dc04 --- /dev/null +++ b/rabbitmq/templates/monitoring/prometheus/exporter-network-policy.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.monitoring.prometheus.network_policy_exporter .Values.monitoring.prometheus.enabled -}} +{{- $netpol_opts := dict "envAll" . "name" "application" "label" "prometheus_rabbitmq_exporter" -}} +{{ $netpol_opts | include "helm-toolkit.manifests.kubernetes_network_policy" }} +{{- end -}} diff --git a/rabbitmq/templates/monitoring/prometheus/exporter-service.yaml b/rabbitmq/templates/monitoring/prometheus/exporter-service.yaml new file mode 100644 index 0000000000..824859adfe --- /dev/null +++ b/rabbitmq/templates/monitoring/prometheus/exporter-service.yaml @@ -0,0 +1,35 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.monitoring.prometheus.service_exporter .Values.monitoring.prometheus.enabled }} +{{- $envAll := . }} +{{- $prometheus_annotations := $envAll.Values.monitoring.prometheus.rabbitmq_exporter }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "prometheus_rabbitmq_exporter" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + labels: +{{ tuple $envAll "prometheus_rabbitmq_exporter" "metrics" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: +{{- if $envAll.Values.monitoring.prometheus.enabled }} +{{ tuple $prometheus_annotations | include "helm-toolkit.snippets.prometheus_service_annotations" | indent 4 }} +{{- end }} +spec: + ports: + - name: metrics + port: {{ tuple "prometheus_rabbitmq_exporter" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + selector: +{{ tuple $envAll "prometheus_rabbitmq_exporter" "exporter" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- end }} diff --git a/rabbitmq/templates/network_policy.yaml b/rabbitmq/templates/network_policy.yaml new file mode 100644 index 0000000000..363b6221fd --- /dev/null +++ b/rabbitmq/templates/network_policy.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} +{{- if .Values.manifests.network_policy -}} +{{- $netpol_opts := dict "envAll" . "name" "application" "label" "rabbitmq" -}} +{{ $netpol_opts | include "helm-toolkit.manifests.kubernetes_network_policy" }} +{{- end -}} diff --git a/rabbitmq/templates/pod-test.yaml b/rabbitmq/templates/pod-test.yaml new file mode 100644 index 0000000000..37d8af3642 --- /dev/null +++ b/rabbitmq/templates/pod-test.yaml @@ -0,0 +1,86 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if (.Values.global).subchart_release_name }} +{{- $_ := set . "deployment_name" .Chart.Name }} +{{- else }} +{{- $_ := set . "deployment_name" .Release.Name }} +{{- end }} + +{{- if .Values.manifests.pod_test }} +{{- $envAll := . }} + +{{ if kindIs "string" $envAll.Values.dependencies.static.tests.jobs }} +{{ if eq $envAll.Values.dependencies.static.tests.jobs "cluster_wait" }} +{{ $_ := set $envAll.Values.dependencies.static.tests "jobs" ( list ( print $envAll.deployment_name "-cluster-wait" ) ) }} +{{ end }} +{{ end }} + +{{- $serviceAccountName := print .deployment_name "-test" }} +{{ tuple $envAll "tests" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} + +{{- $protocol := "http" }} +{{- if $envAll.Values.manifests.certificates }} +{{- $protocol = "https" }} +{{- end }} +--- +apiVersion: v1 +kind: Pod +metadata: + name: "{{.deployment_name}}-test" + labels: +{{ tuple $envAll "rabbitmq" "test" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + "helm.sh/hook": test-success +{{ dict "envAll" $envAll "podName" "rabbitmq-rabbitmq-test" "containerNames" (list "init" "rabbitmq-rabbitmq-test") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 4 }} +spec: +{{ dict "envAll" $envAll "application" "test" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 2 }} + serviceAccountName: {{ $serviceAccountName }} +{{ if $envAll.Values.pod.tolerations.rabbitmq.enabled }} +{{ tuple $envAll "rabbitmq" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 2 }} +{{ end }} + nodeSelector: + {{ $envAll.Values.labels.test.node_selector_key }}: {{ $envAll.Values.labels.test.node_selector_value | quote }} + restartPolicy: Never + initContainers: +{{ tuple $envAll "tests" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 4 }} + containers: + - name: rabbitmq-rabbitmq-test +{{ tuple $envAll "scripted_test" | include "helm-toolkit.snippets.image" | indent 6 }} +{{ dict "envAll" $envAll "application" "test" "container" "rabbitmq_test" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 6 }} + env: + - name: RABBITMQ_ADMIN_CONNECTION + value: {{ tuple "oslo_messaging" "internal" "user" $protocol $envAll | include "helm-toolkit.endpoints.authenticated_endpoint_uri_lookup" | quote }} + - name: RABBIT_REPLICA_COUNT + value: {{ $envAll.Values.pod.replicas.server | quote }} + command: + - /tmp/rabbitmq-test.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: rabbitmq-bin + mountPath: /tmp/rabbitmq-test.sh + subPath: rabbitmq-test.sh + readOnly: true +{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.oslo_messaging.server.internal "path" "/etc/rabbitmq/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 8 }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: rabbitmq-bin + configMap: + name: {{ printf "%s-%s" $envAll.deployment_name "rabbitmq-bin" | quote }} + defaultMode: 0555 +{{- dict "enabled" .Values.manifests.certificates "name" .Values.secrets.tls.oslo_messaging.server.internal | include "helm-toolkit.snippets.tls_volume" | indent 4 }} +{{- end }} diff --git a/rabbitmq/templates/secret-erlang-cookie.yaml b/rabbitmq/templates/secret-erlang-cookie.yaml new file mode 100644 index 0000000000..7022d9ce5a --- /dev/null +++ b/rabbitmq/templates/secret-erlang-cookie.yaml @@ -0,0 +1,31 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if (.Values.global).subchart_release_name }} +{{- $_ := set . "deployment_name" .Chart.Name }} +{{- else }} +{{- $_ := set . "deployment_name" .Release.Name }} +{{- end }} + +{{- if .Values.manifests.secret_erlang_cookie }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ printf "%s-%s" $envAll.deployment_name "erlang-cookie" | quote }} +type: Opaque +data: + erlang_cookie: {{ $envAll.Values.endpoints.oslo_messaging.auth.erlang_cookie | b64enc -}} +{{- end }} diff --git a/rabbitmq/templates/secret-rabbit-admin.yaml b/rabbitmq/templates/secret-rabbit-admin.yaml new file mode 100644 index 0000000000..c80f1bc781 --- /dev/null +++ b/rabbitmq/templates/secret-rabbit-admin.yaml @@ -0,0 +1,33 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if (.Values.global).subchart_release_name }} +{{- $_ := set . "deployment_name" .Chart.Name }} +{{- else }} +{{- $_ := set . "deployment_name" .Release.Name }} +{{- end }} + +{{- if .Values.manifests.secret_admin_user }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ printf "%s-%s" $envAll.deployment_name "admin-user" | quote }} +type: Opaque +data: + RABBITMQ_ADMIN_USERNAME: {{ $envAll.Values.endpoints.oslo_messaging.auth.user.username | b64enc }} + RABBITMQ_ADMIN_PASSWORD: {{ $envAll.Values.endpoints.oslo_messaging.auth.user.password | b64enc }} + RABBITMQ_GUEST_PASSWORD: {{ $envAll.Values.endpoints.oslo_messaging.auth.guest.password | b64enc }} +{{- end }} diff --git a/rabbitmq/templates/secret-rabbitmq-users-credentials.yaml b/rabbitmq/templates/secret-rabbitmq-users-credentials.yaml new file mode 100644 index 0000000000..fc0bf48323 --- /dev/null +++ b/rabbitmq/templates/secret-rabbitmq-users-credentials.yaml @@ -0,0 +1,30 @@ +{{/* +Copyright 2019 Mirantis Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.conf.users }} + +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ printf "%s-%s" $envAll.deployment_name "users-credentials" | quote }} + labels: +{{ tuple $envAll "rabbitmq" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +type: Opaque +data: + RABBITMQ_USERS: {{ toJson .Values.conf.users | b64enc }} +{{- end }} diff --git a/rabbitmq/templates/secret-registry.yaml b/rabbitmq/templates/secret-registry.yaml new file mode 100644 index 0000000000..da979b3223 --- /dev/null +++ b/rabbitmq/templates/secret-registry.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_registry .Values.endpoints.oci_image_registry.auth.enabled }} +{{ include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) }} +{{- end }} diff --git a/rabbitmq/templates/service-ingress-management.yaml b/rabbitmq/templates/service-ingress-management.yaml new file mode 100644 index 0000000000..793ced3cb9 --- /dev/null +++ b/rabbitmq/templates/service-ingress-management.yaml @@ -0,0 +1,29 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if (.Values.global).subchart_release_name }} +{{- $_ := set . "deployment_name" .Chart.Name }} +{{- else }} +{{- $_ := set . "deployment_name" .Release.Name }} +{{- end }} + +{{- if and .Values.manifests.service_ingress_management .Values.network.management.ingress.public }} +{{- $envAll := . }} +{{- if empty $envAll.Values.endpoints.oslo_messaging.hosts.public }} +{{- $service_public_name := .deployment_name | trunc 12 }} +{{- $_ := set $envAll.Values.endpoints.oslo_messaging.hosts "public" ( printf "%s-%s-%s" $service_public_name "mgr" ( $service_public_name | sha256sum | trunc 6 )) }} +{{- end }} +{{- $serviceIngressOpts := dict "envAll" . "backendService" "management" "backendServiceType" "oslo_messaging" "backendPort" "http" -}} +{{ $serviceIngressOpts | include "helm-toolkit.manifests.service_ingress" }} +{{- end }} diff --git a/rabbitmq/templates/service.yaml b/rabbitmq/templates/service.yaml new file mode 100644 index 0000000000..ed7d0dba10 --- /dev/null +++ b/rabbitmq/templates/service.yaml @@ -0,0 +1,41 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.service }} +{{- $envAll := . }} +{{- $protocol := "http" }} +{{- if $envAll.Values.manifests.certificates }} +{{- $protocol = "https" }} +{{- end }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "oslo_messaging" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + labels: +{{ tuple $envAll "rabbitmq" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + clusterIP: None + ports: + - port: {{ tuple "oslo_messaging" "internal" "amqp" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + name: amqp + - port: {{ add (tuple "oslo_messaging" "internal" "amqp" . | include "helm-toolkit.endpoints.endpoint_port_lookup") 20000 }} + name: clustering + - port: {{ tuple "oslo_messaging" "internal" $protocol . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + name: {{ printf "%s" $protocol }} + - name: metrics + port: {{ tuple "oslo_messaging" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + selector: +{{ tuple $envAll "rabbitmq" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{ end }} diff --git a/rabbitmq/templates/statefulset.yaml b/rabbitmq/templates/statefulset.yaml new file mode 100644 index 0000000000..771c5ff3ce --- /dev/null +++ b/rabbitmq/templates/statefulset.yaml @@ -0,0 +1,362 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if (.Values.global).subchart_release_name }} +{{- $_ := set . "deployment_name" .Chart.Name }} +{{- else }} +{{- $_ := set . "deployment_name" .Release.Name }} +{{- end }} + +{{- define "rabbitmqReadinessProbeTemplate" }} +exec: + command: + - /tmp/rabbitmq-readiness.sh +{{- end }} +{{- define "rabbitmqLivenessProbeTemplate" }} +exec: + command: + - /tmp/rabbitmq-liveness.sh +{{- end }} + +{{/* +(aostapenko) rounds cpu limit in any permissible format to integer value (min 1) +"100m" -> 1 +"1100m" -> 1 +"10900m" -> 10 +0.3 -> 1 +5.4 -> 5 +*/}} +{{- define "get_erlvm_scheduler_num" -}} +{{- $val := . | toString -}} +{{- if regexMatch "^[0-9]*m$" $val -}} +{{- $val = div (float64 (trimSuffix "m" $val)) 1000 -}} +{{- end -}} +{{/* NOTE(aostapenko) String with floating number does not convert well to int */}} +{{- $val | float64 | int | default 1 -}} +{{- end -}} + +{{- if .Values.manifests.statefulset }} +{{- $envAll := . }} + +{{- $rcControllerName := printf "%s-%s" $envAll.deployment_name "rabbitmq" }} +{{ tuple $envAll "rabbitmq" $rcControllerName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} + +{{- $protocol := "http" }} +{{- if $envAll.Values.manifests.certificates }} +{{- $protocol = "https" }} +{{- end }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ $rcControllerName | quote }} + namespace: {{ .Release.Namespace }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ $rcControllerName | quote }} +subjects: + - kind: ServiceAccount + name: {{ $rcControllerName | quote }} + namespace: {{ .Release.Namespace }} +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ $rcControllerName | quote }} + namespace: {{ .Release.Namespace }} +rules: + - apiGroups: + - "" + - extensions + - batch + - apps + verbs: + - get + - list + resources: + - services + - endpoints +--- +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ $rcControllerName | quote }} + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "rabbitmq" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + serviceName: {{ tuple "oslo_messaging" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + replicas: {{ $envAll.Values.pod.replicas.server }} + podManagementPolicy: "Parallel" + selector: + matchLabels: +{{ tuple $envAll "rabbitmq" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} + template: + metadata: + labels: +{{ tuple $envAll "rabbitmq" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} + secret-rabbit-admin-hash: {{ tuple "secret-rabbit-admin.yaml" . | include "helm-toolkit.utils.hash" }} + secret-erlang-cookie-hash: {{ tuple "secret-erlang-cookie.yaml" . | include "helm-toolkit.utils.hash" }} +{{ dict "envAll" $envAll "podName" "rabbitmq" "containerNames" (list "init" "rabbitmq-password" "rabbitmq-cookie" "rabbitmq-perms" "rabbitmq") | include "helm-toolkit.snippets.kubernetes_mandatory_access_control_annotation" | indent 8 }} + spec: +{{ dict "envAll" $envAll "application" "server" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $rcControllerName | quote }} + affinity: +{{ tuple $envAll "rabbitmq" "server" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} +{{ if $envAll.Values.pod.tolerations.rabbitmq.enabled }} +{{ tuple $envAll "rabbitmq" | include "helm-toolkit.snippets.kubernetes_tolerations" | indent 6 }} +{{ end }} + nodeSelector: + {{ $envAll.Values.labels.server.node_selector_key }}: {{ $envAll.Values.labels.server.node_selector_value | quote }} + initContainers: +{{ tuple $envAll "rabbitmq" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + - name: rabbitmq-password +{{ tuple $envAll "rabbitmq_init" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "server" "container" "rabbitmq_password" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/rabbitmq-password-hash.py + env: + - name: RABBITMQ_ADMIN_USERNAME + valueFrom: + secretKeyRef: + name: {{ printf "%s-%s" $envAll.deployment_name "admin-user" | quote }} + key: RABBITMQ_ADMIN_USERNAME + - name: RABBITMQ_ADMIN_PASSWORD + valueFrom: + secretKeyRef: + name: {{ printf "%s-%s" $envAll.deployment_name "admin-user" | quote }} + key: RABBITMQ_ADMIN_PASSWORD + - name: RABBITMQ_GUEST_PASSWORD + valueFrom: + secretKeyRef: + name: {{ printf "%s-%s" $envAll.deployment_name "admin-user" | quote }} + key: RABBITMQ_GUEST_PASSWORD + - name: RABBITMQ_DEFINITION_FILE + value: "{{ index $envAll.Values.conf.rabbitmq "management.load_definitions" }}" +{{- if .Values.conf.users }} + - name: RABBITMQ_USERS + valueFrom: + secretKeyRef: + name: {{ printf "%s-%s" $envAll.deployment_name "users-credentials" | quote }} + key: RABBITMQ_USERS +{{- end }} +{{- if .Values.conf.aux_conf }} + - name: RABBITMQ_AUXILIARY_CONFIGURATION + value: {{ toJson $envAll.Values.conf.aux_conf | quote }} +{{- end }} + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: rabbitmq-data + mountPath: /var/lib/rabbitmq + - name: rabbitmq-bin + mountPath: /tmp/rabbitmq-password-hash.py + subPath: rabbitmq-password-hash.py + readOnly: true + - name: rabbitmq-cookie +{{ tuple $envAll "rabbitmq" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "server" "container" "rabbitmq_cookie" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/rabbitmq-cookie.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: rabbitmq-bin + mountPath: /tmp/rabbitmq-cookie.sh + subPath: rabbitmq-cookie.sh + readOnly: true + - name: rabbitmq-data + mountPath: /var/lib/rabbitmq + - name: rabbitmq-erlang-cookie + mountPath: /var/run/lib/rabbitmq/.erlang.cookie + subPath: erlang_cookie + readOnly: true +{{- if $envAll.Values.volume.chown_on_start }} + - name: rabbitmq-perms +{{ tuple $envAll "rabbitmq" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "server" "container" "rabbitmq_perms" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - chown + - -R + - "{{ $envAll.Values.pod.security_context.server.pod.runAsUser }}" + - /var/lib/rabbitmq + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: rabbitmq-data + mountPath: /var/lib/rabbitmq +{{- end }} + containers: + - name: rabbitmq +{{ tuple $envAll "rabbitmq" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "server" "container" "rabbitmq" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/rabbitmq-start.sh + ports: + - name: {{ printf "%s" $protocol }} + protocol: TCP + containerPort: {{ tuple "oslo_messaging" "internal" $protocol . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + {{- if .Values.network.host_namespace }} + hostPort: {{ tuple "oslo_messaging" "internal" $protocol . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + {{- end }} + - name: amqp + protocol: TCP + containerPort: {{ tuple "oslo_messaging" "internal" "amqp" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + {{- if .Values.network.host_namespace }} + hostPort: {{ tuple "oslo_messaging" "internal" "amqp" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + {{- end }} + - name: clustering + protocol: TCP + containerPort: {{ add (tuple "oslo_messaging" "internal" "amqp" . | include "helm-toolkit.endpoints.endpoint_port_lookup") 20000 }} + {{- if .Values.network.host_namespace }} + hostPort: {{ add (tuple "oslo_messaging" "internal" "amqp" . | include "helm-toolkit.endpoints.endpoint_port_lookup") 20000 }} + {{- end }} + - name: metrics + containerPort: {{ tuple "oslo_messaging" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + protocol: TCP + env: + - name: MY_POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: MY_POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: RABBITMQ_USE_LONGNAME + value: "true" + - name: RABBITMQ_NODENAME + value: "rabbit@$(MY_POD_NAME).{{ tuple "oslo_messaging" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }}" + - name: K8S_SERVICE_NAME + value: {{ tuple "oslo_messaging" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} + - name: K8S_HOSTNAME_SUFFIX + value: ".{{ tuple "oslo_messaging" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }}" + - name: RABBITMQ_ERLANG_COOKIE + value: "{{ $envAll.Values.endpoints.oslo_messaging.auth.erlang_cookie }}" + - name: PORT_HTTP + value: "{{ tuple "oslo_messaging" "internal" $protocol . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}" + - name: PORT_AMPQ + value: "{{ tuple "oslo_messaging" "internal" "amqp" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}" + - name: PORT_CLUSTERING + value: "{{ add (tuple "oslo_messaging" "internal" "amqp" . | include "helm-toolkit.endpoints.endpoint_port_lookup") 20000 }}" +{{- if ne (.Values.conf.feature_flags | default "") "default" }} + - name: RABBITMQ_FEATURE_FLAGS + value: "{{ .Values.conf.feature_flags }}" +{{- end }} +{{ dict "envAll" $envAll "component" "rabbitmq" "container" "rabbitmq" "type" "readiness" "probeTemplate" (include "rabbitmqReadinessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | trim | indent 10 }} +{{ dict "envAll" $envAll "component" "rabbitmq" "container" "rabbitmq" "type" "liveness" "probeTemplate" (include "rabbitmqLivenessProbeTemplate" $envAll | fromYaml) | include "helm-toolkit.snippets.kubernetes_probe" | trim | indent 10 }} + lifecycle: + preStop: + exec: + command: + - rabbitmqctl + - stop_app + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: rabbitmq-data + mountPath: /var/lib/rabbitmq + - name: rabbitmq-bin + mountPath: /tmp/rabbitmq-start.sh + subPath: rabbitmq-start.sh + readOnly: true + - name: rabbitmq-bin + mountPath: /tmp/rabbitmq-readiness.sh + subPath: rabbitmq-readiness.sh + readOnly: true + - name: rabbitmq-bin + mountPath: /tmp/rabbitmq-liveness.sh + subPath: rabbitmq-liveness.sh + readOnly: true + - name: rabbitmq-etc + mountPath: /etc/rabbitmq/enabled_plugins + subPath: enabled_plugins + readOnly: true + - name: rabbitmq-etc + mountPath: /etc/rabbitmq/rabbitmq.conf + subPath: rabbitmq.conf + readOnly: true +{{- if .Values.conf.rabbit_advanced_config.enabled }} + - name: rabbitmq-etc + mountPath: /etc/rabbitmq/advanced.config + subPath: advanced.config + readOnly: true +{{- end }} + - name: rabbitmq-etc + mountPath: /etc/rabbitmq/rabbitmq-env.conf + subPath: rabbitmq-env.conf + readOnly: true +{{- if .Values.manifests.config_ipv6 }} + - name: rabbitmq-etc + mountPath: /etc/rabbitmq/erl_inetrc + subPath: erl_inetrc + readOnly: true +{{- end }} +{{- if not .Values.conf.prometheus_exporter.rabbitmq_mgmt_metrics_collector_disabled }} + - name: rabbitmq-etc + mountPath: /etc/rabbitmq/conf.d/management_agent.disable_metrics_collector.conf + subPath: management_agent.disable_metrics_collector.conf + readOnly: true +{{- end }} +{{ dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.secrets.tls.oslo_messaging.server.internal "path" "/etc/rabbitmq/certs" | include "helm-toolkit.snippets.tls_volume_mount" | indent 12 }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: rabbitmq-bin + configMap: + name: {{ printf "%s-%s" $envAll.deployment_name "rabbitmq-bin" | quote }} + defaultMode: 0555 + - name: rabbitmq-etc + configMap: + name: {{ printf "%s-%s" $envAll.deployment_name "rabbitmq-etc" | quote }} + defaultMode: 0444 + - name: rabbitmq-erlang-cookie + secret: + secretName: {{ printf "%s-%s" $envAll.deployment_name "erlang-cookie" | quote }} + defaultMode: 0444 +{{ dict "enabled" $envAll.Values.manifests.certificates "name" $envAll.Values.secrets.tls.oslo_messaging.server.internal | include "helm-toolkit.snippets.tls_volume" | indent 8 }} + {{- if not $envAll.Values.volume.enabled }} + - name: rabbitmq-data + {{- if .Values.volume.use_local_path.enabled }} + hostPath: + path: {{ .Values.volume.use_local_path.host_path }} + type: DirectoryOrCreate + {{- else }} + emptyDir: {} + {{- end }} + {{- end }} +{{- if $envAll.Values.volume.enabled }} + volumeClaimTemplates: + - metadata: + name: rabbitmq-data + spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: {{ $envAll.Values.volume.size }} + {{- if ne .Values.volume.class_name "default" }} + storageClassName: {{ $envAll.Values.volume.class_name }} + {{- end }} +{{- end }} +{{ end }} diff --git a/rabbitmq/templates/utils/_to_rabbit_config.tpl b/rabbitmq/templates/utils/_to_rabbit_config.tpl new file mode 100644 index 0000000000..2adff35410 --- /dev/null +++ b/rabbitmq/templates/utils/_to_rabbit_config.tpl @@ -0,0 +1,35 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- define "rabbitmq.utils.to_rabbit_config" -}} +{{- range $top_key, $top_value := . }} +{{- if kindIs "map" $top_value -}} +{{- range $second_key, $second_value := . }} +{{- if kindIs "map" $second_value -}} +{{- range $third_key, $third_value := . }} +{{- if kindIs "map" $third_value -}} +{{ $top_key }}.{{ $second_key }}.{{ $third_key }} = wow +{{ else -}} +{{ $top_key }}.{{ $second_key }}.{{ $third_key }} = {{ $third_value }} +{{ end -}} +{{- end -}} +{{ else -}} +{{ $top_key }}.{{ $second_key }} = {{ $second_value }} +{{ end -}} +{{- end -}} +{{ else -}} +{{ $top_key }} = {{ $top_value }} +{{ end -}} +{{- end -}} +{{- end -}} diff --git a/rabbitmq/values.yaml b/rabbitmq/values.yaml new file mode 100644 index 0000000000..bc2342fda4 --- /dev/null +++ b/rabbitmq/values.yaml @@ -0,0 +1,495 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for rabbitmq. +# This is a YAML-formatted file. +# Declare name/value pairs to be passed into your templates. +# name: value + +--- +labels: + server: + node_selector_key: openstack-control-plane + node_selector_value: enabled + prometheus_rabbitmq_exporter: + node_selector_key: openstack-control-plane + node_selector_value: enabled + test: + node_selector_key: openstack-control-plane + node_selector_value: enabled + jobs: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +images: + tags: + prometheus_rabbitmq_exporter: docker.io/kbudde/rabbitmq-exporter:v1.0.0-RC7.1 + prometheus_rabbitmq_exporter_helm_tests: docker.io/openstackhelm/heat:2023.2-ubuntu_jammy + rabbitmq_init: docker.io/openstackhelm/heat:2023.2-ubuntu_jammy + rabbitmq: docker.io/library/rabbitmq:3.13.0 + dep_check: quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal + scripted_test: docker.io/library/rabbitmq:3.13.0-management + image_repo_sync: docker.io/library/docker:17.07.0 + pull_policy: "IfNotPresent" + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +# forceBoot: executes 'rabbitmqctl force_boot' to force boot on +# cluster shut down unexpectedly in an unknown order. +# ref: https://www.rabbitmq.com/rabbitmqctl.8.html#force_boot +forceBoot: + enabled: false + +pod: + probes: + prometheus_rabbitmq_exporter: + rabbitmq_exporter: + readiness: + enabled: true + params: + initialDelaySeconds: 30 + periodSeconds: 30 + timeoutSeconds: 5 + liveness: + enabled: true + params: + initialDelaySeconds: 120 + periodSeconds: 90 + timeoutSeconds: 5 + rabbitmq: + rabbitmq: + readiness: + enabled: true + params: + initialDelaySeconds: 10 + periodSeconds: 10 + timeoutSeconds: 10 + successThreshold: 1 + failureThreshold: 3 + liveness: + enabled: true + params: + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 10 + successThreshold: 1 + failureThreshold: 5 + security_context: + exporter: + pod: + runAsUser: 65534 + container: + rabbitmq_exporter: + readOnlyRootFilesystem: true + allowPrivilegeEscalation: false + server: + pod: + runAsUser: 999 + container: + rabbitmq_password: + runAsUser: 0 + readOnlyRootFilesystem: true + rabbitmq_cookie: + runAsUser: 0 + readOnlyRootFilesystem: true + rabbitmq_perms: + runAsUser: 0 + readOnlyRootFilesystem: true + rabbitmq: + allowPrivilegeEscalation: false + runAsUser: 999 + readOnlyRootFilesystem: false + cluster_wait: + pod: + runAsUser: 999 + container: + rabbitmq_cluster_wait: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + rabbitmq_cookie: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + test: + pod: + runAsUser: 999 + container: + rabbitmq_test: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + weight: + default: 10 + tolerations: + rabbitmq: + enabled: false + tolerations: + - key: node-role.kubernetes.io/master + operator: Exists + effect: NoSchedule + - key: node-role.kubernetes.io/control-plane + operator: Exists + effect: NoSchedule + replicas: + server: 2 + prometheus_rabbitmq_exporter: 1 + lifecycle: + upgrades: + deployments: + revision_history: 3 + pod_replacement_strategy: RollingUpdate + rolling_update: + max_unavailable: 1 + max_surge: 3 + termination_grace_period: + prometheus_rabbitmq_exporter: + timeout: 30 + disruption_budget: + mariadb: + min_available: 0 + resources: + enabled: false + prometheus_rabbitmq_exporter: + limits: + memory: "1024Mi" + cpu: "2000m" + requests: + memory: "128Mi" + cpu: "500m" + server: + limits: + memory: "128Mi" + cpu: "500m" + requests: + memory: "128Mi" + cpu: "500m" + jobs: + tests: + limits: + memory: "1024Mi" + cpu: "2000m" + requests: + memory: "128Mi" + cpu: "100m" + image_repo_sync: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + +conf: + enabled_plugins: + - rabbitmq_management + - rabbitmq_peer_discovery_k8s + prometheus_exporter: + capabilities: + - no_sort + log_level: info + skipverify: 1 + skip_queues: "^$" + include_queues: ".*" + rabbit_exporters: "overview,exchange,node,queue" + rabbitmq_mgmt_metrics_collector_disabled: false + # This IP could be IPv4/IPv6 and the tcp port will be appended to it and eventually it is set to rabbitmq.listeners.tcp.1 + bind_address: "::" + rabbitmq: + listeners: + tcp: + # NOTE(portdirect): This is always defined via the endpoints section. + 1: null + cluster_formation: + peer_discovery_backend: rabbit_peer_discovery_k8s + k8s: + address_type: hostname + node_cleanup: + interval: "10" + only_log_warning: "true" + cluster_partition_handling: autoheal + queue_master_locator: min-masters + loopback_users.guest: "false" + management.load_definitions: "/var/lib/rabbitmq/definitions.json" + rabbit_additonal_conf: + # This confinguration is used for non TLS deployments + management.listener.ip: "::" + management.listener.port: null + rabbit_advanced_config: + enabled: false + default_consumer_prefetch: 250 + rabbitmq_exporter: + rabbit_timeout: 30 + # Feature Flags is introduced in RabbitMQ 3.8.0 + # To deploy with standard list of feature, leave as default + # To deploy with specific feature, separate each feature with comma + # To deploy with all features disabled, leave blank or empty + feature_flags: default + users: {} + # define users in the section below which have to be + # created by rabbitmq at start up stage through definitions.json + # file and enable job_users_create manifest. + # users: + # keystone_service: + # auth: + # keystone_username: + # username: keystone + # password: password + # path: /keystone + aux_conf: {} + # aux_conf can be used to pass additional options to definitions.json, allowed keys are: + # - policies + # - bindings + # - parameters + # - queues + # - exchanges + # vhosts,users and permissions are created in users section of values. + # aux_conf: + # policies: + # - vhost: "keystone" + # name: "ha_ttl_keystone" + # definition: + # #mirror messges to other nodes in rmq cluster + # ha-mode: "all" + # ha-sync-mode: "automatic" + # #70s + # message-ttl: 70000 + # priority: 0 + # apply-to: all + # pattern: '^(?!amq\.).*' +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - rabbitmq-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + prometheus_rabbitmq_exporter: + services: + - endpoint: internal + service: oslo_messaging + prometheus_rabbitmq_exporter_tests: + services: + - endpoint: internal + service: prometheus_rabbitmq_exporter + - endpoint: internal + service: monitoring + rabbitmq: + jobs: null + tests: + services: + - endpoint: internal + service: oslo_messaging + # NOTE (portdirect): this key is somewhat special, if set to the string + # `cluster_wait` then the job dep will be populated with a single value + # containing the generated name for the `cluster_wait` job name. + jobs: cluster_wait + cluster_wait: + services: + - endpoint: internal + service: oslo_messaging + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + +monitoring: + prometheus: + enabled: false + rabbitmq_exporter: + scrape: true + +network: + host_namespace: false + management: + ingress: + public: true + classes: + namespace: "nginx" + cluster: "nginx-cluster" + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / + +secrets: + oci_image_registry: + rabbitmq: rabbitmq-oci-image-registry-key + tls: + oslo_messaging: + server: + internal: rabbitmq-tls-direct + +# typically overridden by environmental +# values, but should include all endpoints +# required by this chart +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + rabbitmq: + username: rabbitmq + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + monitoring: + name: prometheus + namespace: null + hosts: + default: prom-metrics + public: prometheus + host_fqdn_override: + default: null + path: + default: null + scheme: + default: 'http' + port: + api: + default: 9090 + public: 80 + oslo_messaging: + auth: + erlang_cookie: openstack-cookie + user: + username: rabbitmq + password: password + guest: + password: password + hosts: + default: rabbitmq + # NOTE(portdirect): the public host is only used to the management WUI + # If left empty, the release name sha suffixed with mgr, will be used to + # produce an unique hostname. + public: null + host_fqdn_override: + default: null + path: / + scheme: rabbit + port: + clustering: + # NOTE(portdirect): the value for this port is driven by amqp+20000 + # it should not be set manually. + default: null + amqp: + default: 5672 + http: + default: 15672 + public: 80 + metrics: + default: 15692 + prometheus_rabbitmq_exporter: + namespace: null + hosts: + default: rabbitmq-exporter + host_fqdn_override: + default: null + path: + default: /metrics + scheme: + default: 'http' + port: + metrics: + default: 9095 + kube_dns: + namespace: kube-system + name: kubernetes-dns + hosts: + default: kube-dns + host_fqdn_override: + default: null + path: + default: null + scheme: http + port: + dns_tcp: + default: 53 + dns: + default: 53 + protocol: UDP + +network_policy: + prometheus_rabbitmq_exporter: + ingress: + - {} + egress: + - {} + rabbitmq: + ingress: + - {} + egress: + - {} + +volume: + use_local_path: + enabled: false + host_path: /var/lib/rabbitmq + chown_on_start: true + enabled: true + class_name: general + size: 768Mi + +# Hook break for helm2. +# Set helm3_hook to false while using helm2 +helm3_hook: true + +manifests: + certificates: false + configmap_bin: true + configmap_etc: true + config_ipv6: false + ingress_management: true + job_cluster_wait: true + job_image_repo_sync: true + monitoring: + prometheus: + configmap_bin: false + deployment_exporter: false + service_exporter: false + network_policy_exporter: false + network_policy: false + pod_test: true + secret_admin_user: true + secret_erlang_cookie: true + secret_registry: true + service_discovery: true + service_ingress_management: true + service: true + statefulset: true +... diff --git a/redis/Chart.yaml b/redis/Chart.yaml new file mode 100644 index 0000000000..67889609c4 --- /dev/null +++ b/redis/Chart.yaml @@ -0,0 +1,24 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: v4.0.1 +description: OpenStack-Helm Redis +name: redis +version: 2024.2.0 +home: https://github.com/redis/redis +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit + version: ">= 0.1.0" +... diff --git a/redis/templates/configmap-bin.yaml b/redis/templates/configmap-bin.yaml new file mode 100644 index 0000000000..227c9c007f --- /dev/null +++ b/redis/templates/configmap-bin.yaml @@ -0,0 +1,29 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_bin }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: redis-bin +data: + image-repo-sync.sh: | +{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} + redis-test.sh: | +{{ tuple "test/_redis_test.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + python-tests.py: | +{{ tuple "test/_python_redis_tests.py.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} diff --git a/redis/templates/deployment.yaml b/redis/templates/deployment.yaml new file mode 100644 index 0000000000..7a2074f182 --- /dev/null +++ b/redis/templates/deployment.yaml @@ -0,0 +1,63 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.deployment }} +{{- $envAll := . }} + +{{- $serviceAccountName := "redis" }} +{{ tuple $envAll "redis" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: redis + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "redis" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + replicas: {{ .Values.pod.replicas.server }} + selector: + matchLabels: +{{ tuple $envAll "redis" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} +{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "redis" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + spec: + serviceAccountName: {{ $serviceAccountName }} + affinity: +{{ tuple $envAll "redis" "server" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} + nodeSelector: + {{ .Values.labels.redis.node_selector_key }}: {{ .Values.labels.redis.node_selector_value | quote }} + initContainers: +{{ tuple $envAll "redis" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: redis +{{ tuple $envAll "redis" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.server | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + command: + - redis-server + - --port + - {{ .Values.network.port | quote }} + ports: + - containerPort: {{ .Values.network.port }} + readinessProbe: + tcpSocket: + port: {{ .Values.network.port }} +{{- end }} diff --git a/redis/templates/job-image-repo-sync.yaml b/redis/templates/job-image-repo-sync.yaml new file mode 100644 index 0000000000..716c5765f3 --- /dev/null +++ b/redis/templates/job-image-repo-sync.yaml @@ -0,0 +1,18 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "redis" -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} diff --git a/redis/templates/pod_test.yaml b/redis/templates/pod_test.yaml new file mode 100644 index 0000000000..e7152580c4 --- /dev/null +++ b/redis/templates/pod_test.yaml @@ -0,0 +1,68 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.helm_tests }} +{{- $envAll := . }} + +{{- $serviceAccountName := print .Release.Name "-test" }} +{{ tuple $envAll "tests" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: v1 +kind: Pod +metadata: + name: "{{.Release.Name}}-test" + labels: +{{ tuple $envAll "redis" "test" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: + "helm.sh/hook": test-success + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +spec: + serviceAccountName: {{ $serviceAccountName }} + nodeSelector: + {{ .Values.labels.test.node_selector_key }}: {{ .Values.labels.test.node_selector_value }} + restartPolicy: Never + containers: + - name: {{.Release.Name}}-helm-tests +{{ tuple $envAll "helm_tests" | include "helm-toolkit.snippets.image" | indent 6 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.tests | include "helm-toolkit.snippets.kubernetes_resources" | indent 6 }} + command: + - /tmp/redis-test.sh + env: + - name: REDIS_HOST + value: "redis" + - name: REDIS_PORT + value: "{{ .Values.network.port }}" + - name: REDIS_DB + value: '0' + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: redis-test + mountPath: /tmp/redis-test.sh + subPath: redis-test.sh + - name: redis-python + mountPath: /tmp/python-tests.py + subPath: python-tests.py + volumes: + - name: pod-tmp + emptyDir: {} + - name: redis-test + configMap: + name: redis-bin + defaultMode: 0555 + - name: redis-python + configMap: + name: redis-bin + defaultMode: 0555 +{{- end }} diff --git a/redis/templates/secret-registry.yaml b/redis/templates/secret-registry.yaml new file mode 100644 index 0000000000..da979b3223 --- /dev/null +++ b/redis/templates/secret-registry.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_registry .Values.endpoints.oci_image_registry.auth.enabled }} +{{ include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) }} +{{- end }} diff --git a/redis/templates/service.yaml b/redis/templates/service.yaml new file mode 100644 index 0000000000..55aee7c2f0 --- /dev/null +++ b/redis/templates/service.yaml @@ -0,0 +1,28 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.service }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Service +metadata: + name: redis +spec: + clusterIP: None + ports: + - port: {{ .Values.network.port }} + selector: +{{ tuple $envAll "redis" "server" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +{{- end }} diff --git a/redis/templates/test/_python_redis_tests.py.tpl b/redis/templates/test/_python_redis_tests.py.tpl new file mode 100644 index 0000000000..748b84eb00 --- /dev/null +++ b/redis/templates/test/_python_redis_tests.py.tpl @@ -0,0 +1,54 @@ +import os +import redis + + +class RedisTest(object): + + def __init__(self): + host = os.environ.get('REDIS_HOST', 'redis') + port = os.environ.get('REDIS_PORT', 6379) + db = os.environ.get('REDIS_DB', 0) + self.redis_conn = redis.Redis(host, port, db) + + def test_connection(self): + ping = self.redis_conn.ping() + if not ping: raise Exception('No connection to database') + print("Successfully connected to database") + + def database_info(self): + ip_port = [] + for client in self.redis_conn.client_list(): + ip_port.append(client["addr"]) + print(ip_port) + if not self.redis_conn.client_list(): + raise Exception('Database client list is null') + return ip_port + + def test_insert_delete_data(self): + key = "test" + value = "it's working" + result_set = self.redis_conn.set(key, value) + if not result_set: raise Exception('ERROR: SET command failed') + print("Successfully SET keyvalue pair") + result_get = self.redis_conn.get(key) + if not result_get: raise Exception('ERROR: GET command failed') + print("Successfully GET keyvalue pair") + db_size = self.redis_conn.dbsize() + if db_size <= 0: raise Exception("Database size not valid") + result_delete = self.redis_conn.delete(key) + if not result_delete == 1: raise Exception("Error: Delete command failed") + print("Successfully DELETED keyvalue pair") + + def test_client_kill(self, client_ip_port_list): + for client_ip_port in client_ip_port_list: + result = self.redis_conn.client_kill(client_ip_port) + if not result: raise Exception('Client failed to be removed') + print("Successfully DELETED client") + + +client_ip_port = [] +redis_client = RedisTest() +redis_client.test_connection() +client_ip_port = redis_client.database_info() +redis_client.test_insert_delete_data() +redis_client.test_client_kill(client_ip_port) diff --git a/redis/templates/test/_redis_test.sh.tpl b/redis/templates/test/_redis_test.sh.tpl new file mode 100644 index 0000000000..b289ea947d --- /dev/null +++ b/redis/templates/test/_redis_test.sh.tpl @@ -0,0 +1,10 @@ +#!/bin/bash +set -ex + +echo "Start Redis Test" +echo "Print Environmental variables" +echo $REDIS_HOST +echo $REDIS_PORT +echo $REDIS_DB + +python /tmp/python-tests.py diff --git a/redis/values.yaml b/redis/values.yaml new file mode 100644 index 0000000000..daa3e2be65 --- /dev/null +++ b/redis/values.yaml @@ -0,0 +1,148 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for redis. +# This is a YAML-formatted file. +# Declare name/value pairs to be passed into your templates. +# name: value + +--- +images: + tags: + redis: docker.io/library/redis:4.0.1 + helm_tests: docker.io/redislabs/redis-py:latest + dep_check: quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal + image_repo_sync: docker.io/library/docker:17.07.0 + pull_policy: IfNotPresent + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +pod: + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + weight: + default: 10 + replicas: + server: 1 + lifecycle: + upgrades: + deployments: + revision_history: 3 + pod_replacement_strategy: RollingUpdate + rolling_update: + max_unavailable: 1 + max_surge: 3 + resources: + enabled: false + server: + limits: + memory: "128Mi" + cpu: "500m" + requests: + memory: "128Mi" + cpu: "500m" + jobs: + image_repo_sync: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + tests: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + +labels: + redis: + node_selector_key: openstack-control-plane + node_selector_value: enabled + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + test: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +network: + port: 6379 + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - redis-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + redis: + services: null + +secrets: + oci_image_registry: + redis: redis-oci-image-registry-key + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + redis: + username: redis + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + +manifests: + configmap_bin: true + deployment: true + job_image_repo_sync: true + secret_registry: true + service: true + helm_tests: true +... diff --git a/registry/Chart.yaml b/registry/Chart.yaml new file mode 100644 index 0000000000..af389cb457 --- /dev/null +++ b/registry/Chart.yaml @@ -0,0 +1,28 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: v2.0.0 +description: OpenStack-Helm Docker Registry +name: registry +version: 2024.2.0 +home: https://github.com/kubernetes/ingress +sources: + - https://opendev.org/openstack/openstack-helm +maintainers: + - name: OpenStack-Helm Authors +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit + version: ">= 0.1.0" +... diff --git a/registry/templates/bin/_bootstrap.sh.tpl b/registry/templates/bin/_bootstrap.sh.tpl new file mode 100644 index 0000000000..755fc1f955 --- /dev/null +++ b/registry/templates/bin/_bootstrap.sh.tpl @@ -0,0 +1,25 @@ +#!/bin/sh + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +{{ .Values.bootstrap.script | default "echo 'Not Enabled'" }} + +IFS=',' ; for IMAGE in ${PRELOAD_IMAGES}; do + docker pull ${IMAGE} + docker tag ${IMAGE} ${LOCAL_REPO}/${IMAGE} + docker push ${LOCAL_REPO}/${IMAGE} +done diff --git a/registry/templates/bin/_registry-proxy.sh.tpl b/registry/templates/bin/_registry-proxy.sh.tpl new file mode 100644 index 0000000000..1f6138cd77 --- /dev/null +++ b/registry/templates/bin/_registry-proxy.sh.tpl @@ -0,0 +1,19 @@ +#!/bin/sh + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +exec nginx -g "daemon off;" diff --git a/registry/templates/bin/_registry.sh.tpl b/registry/templates/bin/_registry.sh.tpl new file mode 100644 index 0000000000..8c6e9c388a --- /dev/null +++ b/registry/templates/bin/_registry.sh.tpl @@ -0,0 +1,19 @@ +#!/bin/sh + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +exec registry serve /etc/docker/registry/config.yml diff --git a/registry/templates/configmap-bin.yaml b/registry/templates/configmap-bin.yaml new file mode 100644 index 0000000000..6f0bc5cbc8 --- /dev/null +++ b/registry/templates/configmap-bin.yaml @@ -0,0 +1,29 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_bin }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: registry-bin +data: + bootstrap.sh: | +{{ tuple "bin/_bootstrap.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + registry.sh: | +{{ tuple "bin/_registry.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} + registry-proxy.sh: | +{{ tuple "bin/_registry-proxy.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} diff --git a/registry/templates/configmap-etc.yaml b/registry/templates/configmap-etc.yaml new file mode 100644 index 0000000000..1fa3c75253 --- /dev/null +++ b/registry/templates/configmap-etc.yaml @@ -0,0 +1,36 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_etc }} +{{- $envAll := . }} + +{{- if empty .Values.conf.registry.http.addr -}} +{{ $_ := cat "0.0.0.0" (tuple "docker_registry" "internal" "registry" . | include "helm-toolkit.endpoints.endpoint_port_lookup") | replace " " ":" | set .Values.conf.registry.http "addr" -}} +{{- end -}} + +{{- if empty .Values.conf.registry.redis.addr -}} +{{ $_ := tuple "redis" "internal" "redis" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" | set .Values.conf.registry.redis "addr" -}} +{{- end -}} + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: registry-etc +data: + config.yml: | +{{ toYaml .Values.conf.registry | indent 4 }} + default.conf: | +{{ tuple "etc/_default.conf.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} diff --git a/registry/templates/daemonset-registry-proxy.yaml b/registry/templates/daemonset-registry-proxy.yaml new file mode 100644 index 0000000000..d61e6ddfd4 --- /dev/null +++ b/registry/templates/daemonset-registry-proxy.yaml @@ -0,0 +1,79 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.daemonset_registry_proxy }} +{{- $envAll := . }} + +{{- $serviceAccountName := "docker-registry-proxy" }} +{{ tuple $envAll "registry_proxy" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: docker-registry-proxy + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "docker" "registry-proxy" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + selector: + matchLabels: +{{ tuple $envAll "docker" "registry-proxy" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} + template: + metadata: + labels: +{{ tuple $envAll "docker" "registry-proxy" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} + spec: +{{ dict "envAll" $envAll "application" "registry_proxy" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + nodeSelector: + {{ .Values.labels.registry.node_selector_key }}: {{ .Values.labels.registry.node_selector_value | quote }} + dnsPolicy: {{ .Values.pod.dns_policy }} + hostNetwork: true + initContainers: +{{ tuple $envAll "registry_proxy" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: registry-proxy +{{ tuple $envAll "registry_proxy" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.registry_proxy | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "registry_proxy" "container" "registry_proxy" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + command: + - /tmp/registry-proxy.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: registry-bin + mountPath: /tmp/registry-proxy.sh + subPath: registry-proxy.sh + readOnly: true + - name: registry-etc + mountPath: /etc/nginx/conf.d/default.conf + subPath: default.conf + readOnly: true + volumes: + - name: pod-tmp + emptyDir: {} + - name: registry-bin + configMap: + name: registry-bin + defaultMode: 0555 + - name: registry-etc + configMap: + name: registry-etc + defaultMode: 0444 +{{- end }} diff --git a/registry/templates/deployment-registry.yaml b/registry/templates/deployment-registry.yaml new file mode 100644 index 0000000000..40d4d2e65c --- /dev/null +++ b/registry/templates/deployment-registry.yaml @@ -0,0 +1,89 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.deployment_registry }} +{{- $envAll := . }} + +{{- $serviceAccountName := "docker-registry" }} +{{ tuple $envAll "registry" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: docker-registry + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} + labels: +{{ tuple $envAll "docker" "registry" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} +spec: + replicas: {{ .Values.pod.replicas.registry }} + selector: + matchLabels: +{{ tuple $envAll "docker" "registry" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 6 }} +{{ tuple $envAll | include "helm-toolkit.snippets.kubernetes_upgrades_deployment" | indent 2 }} + template: + metadata: + labels: +{{ tuple $envAll "docker" "registry" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + annotations: +{{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" | indent 8 }} + configmap-bin-hash: {{ tuple "configmap-bin.yaml" . | include "helm-toolkit.utils.hash" }} + configmap-etc-hash: {{ tuple "configmap-etc.yaml" . | include "helm-toolkit.utils.hash" }} + spec: +{{ dict "envAll" $envAll "application" "registry" | include "helm-toolkit.snippets.kubernetes_pod_security_context" | indent 6 }} + serviceAccountName: {{ $serviceAccountName }} + affinity: +{{ tuple $envAll "docker" "registry" | include "helm-toolkit.snippets.kubernetes_pod_anti_affinity" | indent 8 }} + nodeSelector: + {{ .Values.labels.registry.node_selector_key }}: {{ .Values.labels.registry.node_selector_value | quote }} + initContainers: +{{ tuple $envAll "registry" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: registry +{{ tuple $envAll "registry" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.registry | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} +{{ dict "envAll" $envAll "application" "registry" "container" "registry" | include "helm-toolkit.snippets.kubernetes_container_security_context" | indent 10 }} + ports: + - name: d-reg + containerPort: {{ tuple "docker_registry" "internal" "registry" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + command: + - /tmp/registry.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: registry-bin + mountPath: /tmp/registry.sh + subPath: registry.sh + readOnly: true + - name: registry-etc + mountPath: /etc/docker/registry/config.yml + subPath: config.yml + readOnly: true + - name: docker-images + mountPath: {{ .Values.conf.registry.storage.filesystem.rootdirectory }} + volumes: + - name: pod-tmp + emptyDir: {} + - name: registry-bin + configMap: + name: registry-bin + defaultMode: 0555 + - name: registry-etc + configMap: + name: registry-etc + defaultMode: 0444 + - name: docker-images + persistentVolumeClaim: + claimName: docker-images +{{- end }} diff --git a/registry/templates/etc/_default.conf.tpl b/registry/templates/etc/_default.conf.tpl new file mode 100644 index 0000000000..c387fe4cc2 --- /dev/null +++ b/registry/templates/etc/_default.conf.tpl @@ -0,0 +1,28 @@ +# Docker registry proxy for api version 2 + +upstream docker-registry { + server {{ tuple "docker_registry" "internal" "registry" . | include "helm-toolkit.endpoints.host_and_port_endpoint_uri_lookup" }}; +} + +# No client auth or TLS +# TODO(bacongobbler): experiment with authenticating the registry if it's using TLS +server { + listen {{ tuple "docker_registry" "public" "registry" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}; + server_name localhost; + + # disable any limits to avoid HTTP 413 for large image uploads + client_max_body_size 0; + + # required to avoid HTTP 411: see Issue #1486 (https://github.com/docker/docker/issues/1486) + chunked_transfer_encoding on; + + location / { + # Do not allow connections from docker 1.5 and earlier + # docker pre-1.6.0 did not properly set the user agent on ping, catch "Go *" user agents + if ($http_user_agent ~ "^(docker\/1\.(3|4|5(?!\.[0-9]-dev))|Go ).*$" ) { + return 404; + } + + include docker-registry.conf; + } +} diff --git a/registry/templates/job-bootstrap.yaml b/registry/templates/job-bootstrap.yaml new file mode 100644 index 0000000000..8fc3a80129 --- /dev/null +++ b/registry/templates/job-bootstrap.yaml @@ -0,0 +1,73 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.job_bootstrap }} +{{- $envAll := . }} +{{- if .Values.bootstrap.enabled }} + +{{- $serviceAccountName := "docker-bootstrap" }} +{{ tuple $envAll "bootstrap" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: batch/v1 +kind: Job +metadata: + name: docker-bootstrap + labels: +{{ tuple $envAll "docker" "bootstrap" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: + {{ tuple $envAll | include "helm-toolkit.snippets.release_uuid" }} +spec: + template: + metadata: + labels: +{{ tuple $envAll "docker" "bootstrap" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 8 }} + spec: + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + nodeSelector: + {{ .Values.labels.job.node_selector_key }}: {{ .Values.labels.job.node_selector_value | quote }} + initContainers: +{{ tuple $envAll "bootstrap" list | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 8 }} + containers: + - name: docker-bootstrap +{{ tuple $envAll "bootstrap" | include "helm-toolkit.snippets.image" | indent 10 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.bootstrap | include "helm-toolkit.snippets.kubernetes_resources" | indent 10 }} + env: + - name: LOCAL_REPO + value: "localhost:{{ tuple "docker_registry" "public" "registry" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}" + - name: PRELOAD_IMAGES + value: "{{ include "helm-toolkit.utils.joinListWithComma" .Values.bootstrap.preload_images }}" + command: + - /tmp/bootstrap.sh + volumeMounts: + - name: pod-tmp + mountPath: /tmp + - name: registry-bin + mountPath: /tmp/bootstrap.sh + subPath: bootstrap.sh + readOnly: true + - name: docker-socket + mountPath: /var/run/docker.sock + volumes: + - name: pod-tmp + emptyDir: {} + - name: registry-bin + configMap: + name: registry-bin + defaultMode: 0555 + - name: docker-socket + hostPath: + path: /var/run/docker.sock +{{- end }} +{{- end }} diff --git a/registry/templates/pvc-images.yaml b/registry/templates/pvc-images.yaml new file mode 100644 index 0000000000..94c56f20dd --- /dev/null +++ b/registry/templates/pvc-images.yaml @@ -0,0 +1,30 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.pvc_images }} +{{- $envAll := . }} +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: docker-images +spec: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: {{ .Values.volume.size }} + {{- if ne .Values.volume.class_name "default" }} + storageClassName: {{ .Values.volume.class_name }} + {{- end }} +{{- end }} diff --git a/registry/templates/secret-registry.yaml b/registry/templates/secret-registry.yaml new file mode 100644 index 0000000000..da979b3223 --- /dev/null +++ b/registry/templates/secret-registry.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_registry .Values.endpoints.oci_image_registry.auth.enabled }} +{{ include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) }} +{{- end }} diff --git a/registry/templates/service-registry.yaml b/registry/templates/service-registry.yaml new file mode 100644 index 0000000000..d0eaa5db8c --- /dev/null +++ b/registry/templates/service-registry.yaml @@ -0,0 +1,34 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.service_registry }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "docker_registry" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +spec: + ports: + - name: d-reg + port: {{ tuple "docker_registry" "internal" "registry" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + {{ if .Values.network.registry.node_port.enabled }} + nodePort: {{ .Values.network.registry.node_port.port }} + {{ end }} + selector: +{{ tuple $envAll "docker" "registry" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + {{ if .Values.network.registry.node_port.enabled }} + type: NodePort + {{ end }} +{{- end }} diff --git a/registry/values.yaml b/registry/values.yaml new file mode 100644 index 0000000000..e1ec4fe424 --- /dev/null +++ b/registry/values.yaml @@ -0,0 +1,231 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for docker registry. +# This is a YAML-formatted file. +# Declare name/value pairs to be passed into your templates. +# name: value + +--- +labels: + registry: + node_selector_key: openstack-control-plane + node_selector_value: enabled + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +release_group: null + +images: + tags: + registry: docker.io/library/registry:2 + registry_proxy: registry.k8s.io/kube-registry-proxy:0.4 + bootstrap: docker.io/library/docker:17.07.0 + dep_check: quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal + pull_policy: "IfNotPresent" + local_registry: + active: false + exclude: + - dep_check + +volume: + class_name: general + size: 2Gi + +network: + registry: + ingress: + public: false + node_port: + enabled: false + port: 5000 + +conf: + registry: + version: 0.1 + log: + fields: + service: registry + storage: + cache: + blobdescriptor: redis + filesystem: + rootdirectory: /var/lib/registry + http: + secret: not-so-secret-secret + headers: + X-Content-Type-Options: [nosniff] + health: + storagedriver: + enabled: true + interval: 10s + threshold: 3 + redis: + addr: null + +pod: + security_context: + registry_proxy: + pod: + runAsUser: 65534 + container: + registry_proxy: + runAsUser: 0 + readOnlyRootFilesystem: false + registry: + pod: + runAsUser: 65534 + container: + registry: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + affinity: + anti: + type: + default: preferredDuringSchedulingIgnoredDuringExecution + topologyKey: + default: kubernetes.io/hostname + weight: + default: 10 + dns_policy: "ClusterFirstWithHostNet" + replicas: + registry: 1 + lifecycle: + upgrades: + deployments: + revision_history: 3 + pod_replacement_strategy: RollingUpdate + rolling_update: + max_unavailable: 1 + max_surge: 3 + resources: + enabled: false + registry: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + registry_proxy: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + jobs: + bootstrap: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + +bootstrap: + enabled: true + script: + docker info + preload_images: + - quay.io/kozhukalov/kubernetes-entrypoint:v1.0.0 + +dependencies: + static: + bootstrap: + pod: + # NOTE(srwilkers): As the daemonset dependency is currently broken for + # kubernetes 1.16, use the pod dependency and require the same node + # instead for the same result + - requireSameNode: true + labels: + application: docker + component: registry-proxy + services: + - endpoint: internal + service: docker_registry + registry: + services: + - endpoint: internal + service: redis + registry_proxy: + services: + - endpoint: internal + service: docker_registry + +secrets: + oci_image_registry: + registry: registry-oci-image-registry-key + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + default: 5000 + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + registry: + username: registry + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + docker_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + default: 5000 + redis: + namespace: null + hosts: + default: redis + host_fqdn_override: + default: null + port: + redis: + default: 6379 + +manifests: + configmap_bin: true + configmap_etc: true + daemonset_registry_proxy: true + deployment_registry: true + job_bootstrap: true + job_image_repo_sync: true + pvc_images: true + secret_registry: true + service_registry: true +... diff --git a/release.asc b/release.asc new file mode 100644 index 0000000000..d2961c52e7 --- /dev/null +++ b/release.asc @@ -0,0 +1,29 @@ +-----BEGIN PGP PUBLIC KEY BLOCK----- +Version: GnuPG v1 + +mQINBFX4hgkBEADLqn6O+UFp+ZuwccNldwvh5PzEwKUPlXKPLjQfXlQRig1flpCH +E0HJ5wgGlCtYd3Ol9f9+qU24kDNzfbs5bud58BeE7zFaZ4s0JMOMuVm7p8JhsvkU +C/Lo/7NFh25e4kgJpjvnwua7c2YrA44ggRb1QT19ueOZLK5wCQ1mR+0GdrcHRCLr +7Sdw1d7aLxMT+5nvqfzsmbDullsWOD6RnMdcqhOxZZvpay8OeuK+yb8FVQ4sOIzB +FiNi5cNOFFHg+8dZQoDrK3BpwNxYdGHsYIwU9u6DWWqXybBnB9jd2pve9PlzQUbO +eHEa4Z+jPqxY829f4ldaql7ig8e6BaInTfs2wPnHJ+606g2UH86QUmrVAjVzlLCm +nqoGymoAPGA4ObHu9X3kO8viMBId9FzooVqR8a9En7ZE0Dm9O7puzXR7A1f5sHoz +JdYHnr32I+B8iOixhDUtxIY4GA8biGATNaPd8XR2Ca1hPuZRVuIiGG9HDqUEtXhV +fY5qjTjaThIVKtYgEkWMT+Wet3DPPiWT3ftNOE907e6EWEBCHgsEuuZnAbku1GgD +LBH4/a/yo9bNvGZKRaTUM/1TXhM5XgVKjd07B4cChgKypAVHvef3HKfCG2U/DkyA +LjteHt/V807MtSlQyYaXUTGtDCrQPSlMK5TjmqUnDwy6Qdq8dtWN3DtBWQARAQAB +tCpDZXBoLmNvbSAocmVsZWFzZSBrZXkpIDxzZWN1cml0eUBjZXBoLmNvbT6JAjgE +EwECACIFAlX4hgkCGwMGCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJEOhKwsBG +DzmUXdIQAI8YPcZMBWdv489q8CzxlfRIRZ3Gv/G/8CH+EOExcmkVZ89mVHngCdAP +DOYCl8twWXC1lwJuLDBtkUOHXNuR5+Jcl5zFOUyldq1Hv8u03vjnGT7lLJkJoqpG +l9QD8nBqRvBU7EM+CU7kP8+09b+088pULil+8x46PwgXkvOQwfVKSOr740Q4J4nm +/nUOyTNtToYntmt2fAVWDTIuyPpAqA6jcqSOC7Xoz9cYxkVWnYMLBUySXmSS0uxl +3p+wK0lMG0my/gb+alke5PAQjcE5dtXYzCn+8Lj0uSfCk8Gy0ZOK2oiUjaCGYN6D +u72qDRFBnR3jaoFqi03bGBIMnglGuAPyBZiI7LJgzuT9xumjKTJW3kN4YJxMNYu1 +FzmIyFZpyvZ7930vB2UpCOiIaRdZiX4Z6ZN2frD3a/vBxBNqiNh/BO+Dex+PDfI4 +TqwF8zlcjt4XZ2teQ8nNMR/D8oiYTUW8hwR4laEmDy7ASxe0p5aijmUApWq5UTsF ++s/QbwugccU0iR5orksM5u9MZH4J/mFGKzOltfGXNLYI6D5Mtwrnyi0BsF5eY0u6 +vkdivtdqrq2DXY+ftuqLOQ7b+t1RctbcMHGPptlxFuN9ufP5TiTWSpfqDwmHCLsT +k2vFiMwcHdLpQ1IH8ORVRgPPsiBnBOJ/kIiXG2SxPUTjjEGOVgeA +=/Tod +-----END PGP PUBLIC KEY BLOCK----- diff --git a/releasenotes/notes/added-nova-uid-parameter-to-ovs-chart-41d2b05b79300a31.yaml b/releasenotes/notes/added-nova-uid-parameter-to-ovs-chart-41d2b05b79300a31.yaml new file mode 100644 index 0000000000..853d7c71d6 --- /dev/null +++ b/releasenotes/notes/added-nova-uid-parameter-to-ovs-chart-41d2b05b79300a31.yaml @@ -0,0 +1,12 @@ +--- +other: + - | + When running openvswitch (OVS) with DPDK enabled, vhost-user sockets are + used to connect VMs to OVS. nova-compute needs access to those sockets in + order to plug them into OVS. For this reason, the directory containing + vhost-user sockets must have proper permissions. The openvswitch chart now + sets ownership of this directory to the UID of the nova user. The OVS chart + uses the same default as the Nova chart (42424). However, if the Nova UID + is changed in the Nova chart in a particular deployment, it also needs to + be changed in the OVS chart correspondingly if DPDK is used. +... diff --git a/releasenotes/notes/ca-clusterissuer.yaml b/releasenotes/notes/ca-clusterissuer.yaml new file mode 100644 index 0000000000..b69b883fc9 --- /dev/null +++ b/releasenotes/notes/ca-clusterissuer.yaml @@ -0,0 +1,7 @@ +--- +ca-clusterissuer: + - 0.1.0 Initial Chart + - 0.1.1 Update htk requirements + - 0.1.2 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/ca-issuer.yaml b/releasenotes/notes/ca-issuer.yaml new file mode 100644 index 0000000000..c772ccec1a --- /dev/null +++ b/releasenotes/notes/ca-issuer.yaml @@ -0,0 +1,12 @@ +--- +ca-issuer: + - 0.1.0 Initial Chart + - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" + - 0.1.2 Update apiVersion of Issuer to v1 + - 0.1.3 Revert - Update apiVersion of Issuer to v1 + - 0.2.0 Only Cert-manager version v1.0.0 or greater will be supported + - 0.2.1 Cert-manager "< v1.0.0" supports cert-manager.io/v1alpha3 else use api cert-manager.io/v1 + - 0.2.2 Update htk requirements + - 0.2.3 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/ceph-adapter-rook.yaml b/releasenotes/notes/ceph-adapter-rook.yaml new file mode 100644 index 0000000000..25b4046590 --- /dev/null +++ b/releasenotes/notes/ceph-adapter-rook.yaml @@ -0,0 +1,10 @@ +--- +ceph-adapter-rook: + - 0.1.0 Initial Chart + - 0.1.1 Update Ceph images to Jammy and Reef 18.2.1 + - 0.1.2 Update Ceph images to patched 18.2.2 and restore debian-reef repo + - 0.1.3 Simplify and remove unnecessary entities + - 0.1.4 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.1.5 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/ceph-client.yaml b/releasenotes/notes/ceph-client.yaml new file mode 100644 index 0000000000..f90c2be96a --- /dev/null +++ b/releasenotes/notes/ceph-client.yaml @@ -0,0 +1,58 @@ +--- +ceph-client: + - 0.1.0 Initial Chart + - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" + - 0.1.2 fix the logic to disable the autoscaler on pools + - 0.1.3 Run as ceph user and disallow privilege escalation + - 0.1.4 Improvements for ceph-client helm tests + - 0.1.5 Fix Helm test check_pgs() check for inactive PGs + - 0.1.6 Uplift from Nautilus to Octopus release + - 0.1.7 Don't wait for premerge PGs in the rbd pool job + - 0.1.8 enhance logic to enable the autoscaler for Octopus + - 0.1.9 Revert "[ceph-client] enhance logic to enable the autoscaler for Octopus" + - 0.1.10 Separate pool quotas from pg_num calculations + - 0.1.11 enhance logic to enable and disable the autoscaler + - 0.1.12 Disable autoscaling before pools are created + - 0.1.13 Fix ceph-client helm test + - 0.1.14 Allow Ceph RBD pool job to leave failed pods + - 0.1.15 Make ceph-client helm test more PG specific + - 0.1.16 Make Ceph pool init job consistent with helm test + - 0.1.17 Add pool rename support for Ceph pools + - 0.1.18 Add pool delete support for Ceph pools + - 0.1.19 Use full image ref for docker official images + - 0.1.20 Export crash dumps when Ceph daemons crash + - 0.1.21 Fix Ceph checkDNS script + - 0.1.22 Set pg_num_min in all cases + - 0.1.23 Helm 3 - Fix Job labels + - 0.1.24 Performance optimizations for the ceph-rbd-pool job + - 0.1.25 Update htk requirements + - 0.1.26 Fix ceph-rbd-pool deletion race + - 0.1.27 Update ceph_mon config to ips from fqdn + - 0.1.28 Fix ceph.conf update job labels, rendering + - 0.1.29 Consolidate mon_host discovery + - 0.1.30 Move ceph-mgr deployment to the ceph-mon chart + - 0.1.31 Consolidate mon_endpoints discovery + - 0.1.32 Simplify test rules for ceph-mgr deployment + - 0.1.33 More robust naming of clusterrole-checkdns + - 0.1.34 Migrated CronJob resource to batch/v1 API version + - 0.1.35 Handle multiple mon versions in the pool job + - 0.1.36 Add the ability to run Ceph commands from values + - 0.1.37 Added OCI registry authentication + - 0.1.38 Make use of noautoscale with Pacific + - 0.1.39 Correct check for too many OSDs in the pool job + - 0.1.40 Fix OSD count checks in the ceph-rbd-pool job + - 0.1.41 Allow gate scripts to use 1x replication in Ceph + - 0.1.42 Update all Ceph images to Focal + - 0 1.43 Document the use of mon_allow_pool_size_one + - 0.1.44 Allow pg_num_min to be overridden per pool + - 0.1.45 Update Ceph to 17.2.6 + - 0.1.46 Strip any errors preceding pool properties JSON + - 0.1.47 Use Helm toolkit functions for Ceph probes + - 0.1.48 Update Rook to 1.12.5 and Ceph to 18.2.0 + - 0.1.49 Update Ceph images to Jammy and Reef 18.2.1 + - 0.1.50 Update Ceph images to patched 18.2.2 and restore debian-reef repo + - 0.1.51 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.1.52 Run utils-defragOSDs.sh in ceph-osd-default container + - 0.1.53 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/ceph-mon.yaml b/releasenotes/notes/ceph-mon.yaml new file mode 100644 index 0000000000..4dadd84c49 --- /dev/null +++ b/releasenotes/notes/ceph-mon.yaml @@ -0,0 +1,42 @@ +--- +ceph-mon: + - 0.1.0 Initial Chart + - 0.1.1 Change helm-toolkit dependency to >= 0.1.0 + - 0.1.2 Enable shareProcessNamespace in mon daemonset + - 0.1.3 Run mon container as ceph user + - 0.1.4 Uplift from Nautilus to Octopus release + - 0.1.5 Add Ceph CSI plugin + - 0.1.6 Fix python3 issue for util scripts + - 0.1.7 remove deprecated svc annotation tolerate-unready-endpoints + - 0.1.8 Use full image ref for docker official images + - 0.1.9 Remove unnecessary parameters for ceph-mon + - 0.1.10 Export crash dumps when Ceph daemons crash + - 0.1.11 Correct mon-check executing binary and logic + - 0.1.12 Fix Ceph checkDNS script + - 0.1.13 Helm 3 - Fix Job labels + - 0.1.14 Update htk requirements + - 0.1.15 Prevent mon-check from removing mons when down temporarily + - 0.1.16 Correct Ceph Mon Check Ports + - 0.1.17 Skip monmap endpoint check for missing mons + - 0.1.18 Move ceph-mgr deployment to the ceph-mon chart + - 0.1.19 Add a post-apply job to restart mons after mgrs + - 0.1.20 Consolidate mon_endpoints discovery + - 0.1.21 Change configmap names to be based on release name + - 0.1.22 Correct configmap names for all resources + - 0.1.23 Release-specific ceph-template configmap name + - 0.1.24 Prevents mgr SA from repeated creation + - 0.1.25 Allow for unconditional mon restart + - 0.1.26 Added OCI registry authentication + - 0.1.27 Update all Ceph images to Focal + - 0.1.28 Document the use of mon_allow_pool_size_one + - 0.1.29 Update Ceph to 17.2.6 + - 0.1.30 Use Helm tookkit functions for Ceph probes + - 0.1.31 Add Rook Helm charts for managing Ceph with Rook + - 0.1.32 Update Rook to 1.12.5 and Ceph to 18.2.0 + - 0.1.33 Update Ceph images to Jammy and Reef 18.2.1 + - 0.1.34 Update Ceph images to patched 18.2.2 and restore debian-reef repo + - 0.1.35 Use seprate secrets for CSI plugin and CSI provisioner + - 0.1.36 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.1.37 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/ceph-osd.yaml b/releasenotes/notes/ceph-osd.yaml new file mode 100644 index 0000000000..5aeee5b2eb --- /dev/null +++ b/releasenotes/notes/ceph-osd.yaml @@ -0,0 +1,63 @@ +--- +ceph-osd: + - 0.1.0 Initial Chart + - 0.1.1 Change helm-toolkit dependency to >= 0.1.0 + - 0.1.2 wait for only osd pods from post apply job + - 0.1.3 Search for complete logical volume name for OSD data volumes + - 0.1.4 Don't try to prepare OSD disks that are already deployed + - 0.1.5 Fix the sync issue between osds when using shared disk for metadata + - 0.1.6 Logic improvement for used osd disk detection + - 0.1.7 Synchronization audit for the ceph-volume osd-init script + - 0.1.8 Update post apply job + - 0.1.9 Check inactive PGs multiple times + - 0.1.10 Fix typo in check inactive PGs logic + - 0.1.11 Fix post-apply job failure related to fault tolerance + - 0.1.12 Add a check for misplaced objects to the post-apply job + - 0.1.13 Remove default OSD configuration + - 0.1.14 Alias synchronized commands and fix descriptor leak + - 0.1.15 Correct naming convention for logical volumes in disk_zap() + - 0.1.16 dmsetup remove logical devices using correct device names + - 0.1.17 Fix a bug with DB orphan volume removal + - 0.1.18 Uplift from Nautilus to Octopus release + - 0.1.19 Update rbac api version + - 0.1.20 Update directory-based OSD deployment for image changes + - 0.1.21 Refactor Ceph OSD Init Scripts - First PS + - 0.1.22 Refactor Ceph OSD Init Scripts - Second PS + - 0.1.23 Use full image ref for docker official images + - 0.1.24 Ceph OSD Init Improvements + - 0.1.25 Export crash dumps when Ceph daemons crash + - 0.1.26 Mount /var/crash inside ceph-osd pods + - 0.1.27 Limit Ceph OSD Container Security Contexts + - 0.1.28 Change var crash mount propagation to HostToContainer + - 0.1.29 Fix Ceph checkDNS script + - 0.1.30 Ceph OSD log-runner container should run as ceph user + - 0.1.31 Helm 3 - Fix Job labels + - 0.1.32 Update htk requirements + - 0.1.33 Update log-runner container for MAC + - 0.1.34 Remove wait for misplaced objects during OSD restarts + - 0.1.35 Consolidate mon_endpoints discovery + - 0.1.36 Add OSD device location pre-check + - 0.1.37 Add a disruptive OSD restart to the post-apply job + - 0.1.38 Skip pod wait in post-apply job when disruptive + - 0.1.39 Allow for unconditional OSD restart + - 0.1.40 Remove udev interactions from osd-init + - 0.1.41 Remove ceph-mon dependency in ceph-osd liveness probe + - 0.1.42 Added OCI registry authentication + - 0.1.43 Update all Ceph images to Focal + - 0.1.44 Update Ceph to 17.2.6 + - 0.1.45 Extend the ceph-osd post-apply job PG wait + - 0.1.46 Use Helm toolkit functions for Ceph probes + - 0.1.47 Add disk zap to OSD init forced repair case + - 0.1.48 Update Rook to 1.12.5 and Ceph to 18.2.0 + - 0.1.49 Update Ceph images to Jammy and Reef 18.2.1 + - 0.1.50 Allow lvcreate to wipe existing LV metadata + - 0.1.51 Update Ceph images to patched 18.2.2 and restore debian-reef repo + - 0.1.52 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.1.53 Update ceph-daemon to be able to use tini init system + - 0.1.54 Remove use of tini for ceph-daemon + - 0.1.55 Update ceph-osd pod containers to make sure OSD pods are properly terminated at restart + - 0.1.56 Add preStop lifecycle script to log-runner + - 0.1.57 Added code to kill another background process in log-runner at restart + - 0.1.58 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/ceph-provisioners.yaml b/releasenotes/notes/ceph-provisioners.yaml new file mode 100644 index 0000000000..0dd69d1b2c --- /dev/null +++ b/releasenotes/notes/ceph-provisioners.yaml @@ -0,0 +1,38 @@ +--- +ceph-provisioners: + - 0.1.0 Initial Chart + - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" + - 0.1.2 Validate each storageclass created + - 0.1.3 Uplift from Nautilus to Octopus release + - 0.1.4 Add Ceph CSI plugin + - 0.1.5 Fix Helm tests for the Ceph provisioners + - 0.1.6 Update ceph_mon config as per new ceph clients + - 0.1.7 Use full image ref for docker official images + - 0.1.8 Enable Ceph CSI Provisioner to Stand Alone + - 0.1.10 Add check for empty ceph endpoint + - 0.1.11 Limit Ceph Provisioner Container Security Contexts + - 0.1.12 Add ceph mon v2 port for ceph csi provisioner + - 0.1.13 Fix ceph-provisioner rbd-healer error + - 0.1.14 Helm 3 - Fix Job labels + - 0.1.15 Add support to connect to rook-ceph cluster + - 0.1.16 Update htk requirements + - 0.1.17 Consolidate mon_endpoints discovery + - 0.1.18 Update CSI images & fix ceph csi provisioner RBAC + - 0.1.19 Add pods watch and list permissions to cluster role + - 0.1.20 Add missing CRDs for volume snapshots (classes, contents) + - 0.1.21 Added OCI registry authentication + - 0.1.22 Remove legacy Ceph provisioners + - 0.1.23 Remove unnecessary templates + - 0.1.24 Update all Ceph images to Focal + - 0.1.25 Update kubernetes registry to registry.k8s.io + - 0.1.26 Update Ceph to 17.2.6 + - 0.1.27 Update Rook to 1.12.5 and Ceph to 18.2.0 + - 0.1.28 Update Ceph images to Jammy and Reef 18.2.1 + - 0.1.29 Update Ceph images to patched 18.2.2 and restore debian-reef repo + - 0.1.30 Specify CSI drivername in values.yaml + - 0.1.31 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.1.32 Update ceph_rbd_provisioner image to 18.2.2 + - 0.1.33 Remove dependencies on legacy provisioners + - 0.1.34 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/ceph-rgw.yaml b/releasenotes/notes/ceph-rgw.yaml new file mode 100644 index 0000000000..547136a4b9 --- /dev/null +++ b/releasenotes/notes/ceph-rgw.yaml @@ -0,0 +1,44 @@ +--- +ceph-rgw: + - 0.1.0 Initial Chart + - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" + - 0.1.2 Uplift from Nautilus to Octopus release + - 0.1.3 update rbac api version + - 0.1.4 Rgw placement target support + - 0.1.5 Add tls support + - 0.1.6 Update tls override options + - 0.1.7 Use ca cert for helm tests + - 0.1.8 Add placement target delete support to RGW + - 0.1.9 Use full image ref for docker official images + - 0.1.10 Fix a bug in placement target deletion for new targets + - 0.1.11 Change s3 auth order to use local before external + - 0.1.12 Export crash dumps when Ceph daemons crash + - 0.1.13 Add configmap hash for keystone rgw + - 0.1.14 Disable crash dumps for rgw + - 0.1.15 Correct rgw placement target functions + - 0.1.16 Helm 3 - Fix Job labels + - 0.1.17 Update htk requirements + - 0.1.18 Consolidate mon_endpoints discovery + - 0.1.19 Add ClusterRole to the bootstrap-job + - 0.1.20 Enable taint toleration for Openstack services jobs + - 0.1.21 Correct mon discovery for multiple RGWs in different NS + - 0.1.22 Update default image values + - 0.1.23 Added OCI registry authentication + - 0.1.24 Replace civetweb with beast for unencrypted connections + - 0.1.25 Update all Ceph images to Focal + - 0.1.26 Replace node-role.kubernetes.io/master with control-plane + - 0.1.27 Update Ceph to 17.2.6 + - 0.1.28 Use Helm toolkit functions for Ceph probes + - 0.1.29 Add 2023.1 Ubuntu Focal overrides + - 0.1.30 Update Rook to 1.12.5 and Ceph to 18.2.0 + - 0.1.31 Add a ceph-rgw-pool job to manage RGW pools + - 0.1.32 Multiple namespace support for the ceph-rgw-pool job + - 0.1.33 Update Ceph images to Jammy and Reef 18.2.1 + - 0.1.34 Update Ceph images to patched 18.2.2 and restore debian-reef repo + - 0.1.35 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.1.36 Add 2024.1 Ubuntu Jammy overrides + - 0.1.37 Update heat image default tag to 2024.1-ubuntu_jammy + - 0.1.38 Add 2024.2 overrides + - 0.1.39 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/cert-rotation.yaml b/releasenotes/notes/cert-rotation.yaml new file mode 100644 index 0000000000..1242b3eb5d --- /dev/null +++ b/releasenotes/notes/cert-rotation.yaml @@ -0,0 +1,15 @@ +--- +cert-rotation: + - 0.1.0 Initial Chart + - 0.1.1 Return true if grep finds no match + - 0.1.2 Correct and enhance the rotation script + - 0.1.3 Update htk requirements + - 0.1.4 Consider initContainers when restarting resources + - 0.1.5 Migrated CronJob resource to batch/v1 API version + - 0.1.6 Added OCI registry authentication + - 0.1.7 Update all Ceph images to Focal + - 0.1.8 Update Ceph images to Jammy and Reef 18.2.1 + - 0.1.9 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.1.10 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/change-default-ovs-image-c1e24787f1b03170.yaml b/releasenotes/notes/change-default-ovs-image-c1e24787f1b03170.yaml new file mode 100644 index 0000000000..c07024c90a --- /dev/null +++ b/releasenotes/notes/change-default-ovs-image-c1e24787f1b03170.yaml @@ -0,0 +1,8 @@ +--- +other: + - | + The default image used by the openvswitch chart has been changed from a + a Debian based image including a source build of openvswitch v2.8.1 to an + Ubuntu Bionic based image including a distribution provided build of + openvswitch v2.9.2. +... diff --git a/releasenotes/notes/changed-ovs-dpdk-root-key-f8aaf3ad65189c8a.yaml b/releasenotes/notes/changed-ovs-dpdk-root-key-f8aaf3ad65189c8a.yaml new file mode 100644 index 0000000000..795c409359 --- /dev/null +++ b/releasenotes/notes/changed-ovs-dpdk-root-key-f8aaf3ad65189c8a.yaml @@ -0,0 +1,7 @@ +--- +other: + - | + The root configuration key of the DPDK section has been changed from + "dpdk" to "ovs_dpdk" to achieve parity with the corresponding configuration + key in the Neutron chart. +... diff --git a/releasenotes/notes/daemonjob-controller.yaml b/releasenotes/notes/daemonjob-controller.yaml new file mode 100644 index 0000000000..db272a6472 --- /dev/null +++ b/releasenotes/notes/daemonjob-controller.yaml @@ -0,0 +1,13 @@ +--- +daemonjob-controller: + - 0.1.0 Initial Chart + - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" + - 0.1.2 Add default value for property in x-kubernetes-list-map-keys + - 0.1.3 Update to container image repo k8s.gcr.io + - 0.1.4 Use full image ref for docker official images + - 0.1.5 Update htk requirements + - 0.1.6 Added OCI registry authentication + - 0.1.7 Update kubernetes registry to registry.k8s.io + - 0.1.8 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/elastic-apm-server.yaml b/releasenotes/notes/elastic-apm-server.yaml new file mode 100644 index 0000000000..fd80eceb87 --- /dev/null +++ b/releasenotes/notes/elastic-apm-server.yaml @@ -0,0 +1,11 @@ +--- +elastic-apm-server: + - 0.1.0 Initial Chart + - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" + - 0.1.2 Use full image ref for docker official images + - 0.1.3 Update htk requirements + - 0.1.4 Added OCI registry authentication + - 0.1.5 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.1.6 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/elastic-filebeat.yaml b/releasenotes/notes/elastic-filebeat.yaml new file mode 100644 index 0000000000..f41623bed8 --- /dev/null +++ b/releasenotes/notes/elastic-filebeat.yaml @@ -0,0 +1,12 @@ +--- +elastic-filebeat: + - 0.1.0 Initial Chart + - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" + - 0.1.2 Use full image ref for docker official images + - 0.1.3 Update htk requirements + - 0.1.4 Added OCI registry authentication + - 0.1.5 Replace node-role.kubernetes.io/master with control-plane + - 0.1.6 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.1.7 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/elastic-metricbeat.yaml b/releasenotes/notes/elastic-metricbeat.yaml new file mode 100644 index 0000000000..191f794017 --- /dev/null +++ b/releasenotes/notes/elastic-metricbeat.yaml @@ -0,0 +1,13 @@ +--- +elastic-metricbeat: + - 0.1.0 Initial Chart + - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" + - 0.1.2 Update RBAC apiVersion from /v1beta1 to /v1 + - 0.1.3 Use full image ref for docker official images + - 0.1.4 Update htk requirements + - 0.1.5 Added OCI registry authentication + - 0.1.6 Replace node-role.kubernetes.io/master with control-plane + - 0.1.7 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.1.8 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/elastic-packetbeat.yaml b/releasenotes/notes/elastic-packetbeat.yaml new file mode 100644 index 0000000000..66e9ff5335 --- /dev/null +++ b/releasenotes/notes/elastic-packetbeat.yaml @@ -0,0 +1,11 @@ +--- +elastic-packetbeat: + - 0.1.0 Initial Chart + - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" + - 0.1.2 Use full image ref for docker official images + - 0.1.3 Update htk requirements + - 0.1.4 Added OCI registry authentication + - 0.1.5 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.1.6 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/elasticsearch.yaml b/releasenotes/notes/elasticsearch.yaml new file mode 100644 index 0000000000..73352c8851 --- /dev/null +++ b/releasenotes/notes/elasticsearch.yaml @@ -0,0 +1,54 @@ +--- +elasticsearch: + - 0.1.0 Initial Chart + - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" + - 0.1.2 Update to 7.6.2 image + - 0.1.3 Add elasticsearch snapshot policy template for SLM + - 0.1.4 Add elasticsearch ILM functionality + - 0.1.5 Make templates job more generic + - 0.1.6 Fix elasticsearch-master rendering error + - 0.1.7 Pin Java options to specific versions + - 0.1.8 Disable Curator in Gate & Chart Defaults + - 0.2.0 Add more S3 configuration options + - 0.2.1 Make templates job more robust & allow overrides + - 0.2.2 Update the ES curator config to {} + - 0.2.3 Add configurable backoffLimit to templates job + - 0.2.4 Update helm-test script + - 0.2.5 Enable TLS with Kibana + - 0.2.6 Enable TLS path between nodes in cluster and TLS path between ceph-rgw + - 0.2.7 Get connection option from values.yaml + - 0.2.8 Use full image ref for docker official images + - 0.2.9 Removed repo verification check from helm-test + - 0.2.10 Enable TLS path between Prometheus-elasticsearch-exporter and Elasticsearch + - 0.2.11 Enable TLS path between Curator and Elasticsearch + - 0.2.12 Helm 3 - Fix Job labels + - 0.2.13 Update htk requirements + - 0.2.14 Fix cronjob rendering + - 0.2.15 Fix elasticsearch-data shutdown + - 0.2.16 Use python3 for helm tests when possible + - 0.2.17 Annotate ES master/data sts with S3 secret hash + - 0.2.18 Update default image value to Wallaby + - 0.2.19 Migrated CronJob resource to batch/v1 API version + - 0.2.20 Set default python for helm test + - 0.2.21 Added OCI registry authentication + - 0.2.22 Update all Ceph images to Focal + - 0.2.23 Add configurable liveness probe for elasticsearch client + - 0.2.24 Update Ceph to 17.2.6 + - 0.2.25 Update ElasticSearch to 8.9.0 + - 0.2.26 Add 2023.1 Ubuntu Focal overrides + - 0.2.27 Update Rook to 1.12.5 and Ceph to 18.2.0 + - 0.2.28 Utilize bucket claim CRD when using with Rook + - 0.2.29 Make es curator path configurable + - 0.2.30 Update curator for es v8 + - 0.3.0 Update elasticsearch_exporter to v1.7.0 + - 0.3.1 Update Ceph images to Jammy and Reef 18.2.1 + - 0.3.2 Update Ceph images to patched 18.2.2 and restore debian-reef repo + - 0.3.3 Update es curator to 8.0.10 + - 0.3.4 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.3.5 Remove gateway node role + - 0.3.6 Add 2024.1 Ubuntu Jammy overrides + - 0.3.7 Add 2024.2 overrides + - 0.3.8 Remove use of python in helm tests + - 0.3.9 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/etcd.yaml b/releasenotes/notes/etcd.yaml new file mode 100644 index 0000000000..cd27770e68 --- /dev/null +++ b/releasenotes/notes/etcd.yaml @@ -0,0 +1,15 @@ +--- +etcd: + - 0.1.0 Initial Chart + - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" + - 0.1.2 Update to container image repo k8s.gcr.io + - 0.1.3 Use full image ref for docker official images + - 0.1.4 Update htk requirements + - 0.1.5 Added OCI registry authentication + - 0.1.6 Update kubernetes registry to registry.k8s.io + - 0.1.7 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.1.8 Switch etcd to staetefulset + - 0.1.9 Adding cronjob with etcd compaction + - 0.1.10 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/falco.yaml b/releasenotes/notes/falco.yaml new file mode 100644 index 0000000000..2da3f34d7a --- /dev/null +++ b/releasenotes/notes/falco.yaml @@ -0,0 +1,16 @@ +--- +falco: + - 0.1.0 Initial Chart + - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" + - 0.1.2 Update to container image repo k8s.gcr.io + - 0.1.3 Remove zookeeper residue + - 0.1.4 Remove kafka residue + - 0.1.5 Use full image ref for docker official images + - 0.1.6 Update htk requirements + - 0.1.7 Added OCI registry authentication + - 0.1.8 Replace node-role.kubernetes.io/master with control-plane + - 0.1.9 Update kubernetes registry to registry.k8s.io + - 0.1.10 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.1.11 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/flannel.yaml b/releasenotes/notes/flannel.yaml new file mode 100644 index 0000000000..86976be099 --- /dev/null +++ b/releasenotes/notes/flannel.yaml @@ -0,0 +1,12 @@ +--- +flannel: + - 0.1.0 Initial Chart + - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" + - 0.1.2 Use full image ref for docker official images + - 0.1.3 Update htk requirements + - 0.1.4 Added OCI registry authentication + - 0.1.5 Replace node-role.kubernetes.io/master with control-plane + - 0.1.6 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.1.7 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/fluentbit.yaml b/releasenotes/notes/fluentbit.yaml new file mode 100644 index 0000000000..343d84d210 --- /dev/null +++ b/releasenotes/notes/fluentbit.yaml @@ -0,0 +1,12 @@ +--- +fluentbit: + - 0.1.0 Initial Chart + - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" + - 0.1.2 Use full image ref for docker official images + - 0.1.3 Update htk requirements + - 0.1.4 Added OCI registry authentication + - 0.1.5 Replace node-role.kubernetes.io/master with control-plane + - 0.1.6 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.1.7 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/fluentd.yaml b/releasenotes/notes/fluentd.yaml new file mode 100644 index 0000000000..224080117f --- /dev/null +++ b/releasenotes/notes/fluentd.yaml @@ -0,0 +1,19 @@ +--- +fluentd: + - 0.1.0 Initial Chart + - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" + - 0.1.2 Add Configurable Readiness and Liveness Probes + - 0.1.3 Enable TLS path for output to Elasticsearch + - 0.1.4 Use full image ref for docker official images + - 0.1.5 Kafka brokers defined as a list with port "kafka1:9092,kafka2:9020,kafka3:9092" + - 0.1.6 Update htk requirements + - 0.1.7 Update default image values to Wallaby + - 0.1.8 Added OCI registry authentication + - 0.1.9 Set sticky bit for tmp + - 0.1.10 Add 2023.1 Ubuntu Focal overrides + - 0.1.11 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.1.12 Add 2024.1 Ubuntu Jammy overrides + - 0.1.13 Add 2024.2 overrides + - 0.1.14 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/gnocchi.yaml b/releasenotes/notes/gnocchi.yaml new file mode 100644 index 0000000000..bab0b6db69 --- /dev/null +++ b/releasenotes/notes/gnocchi.yaml @@ -0,0 +1,22 @@ +--- +gnocchi: + - 0.1.0 Initial Chart + - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" + - 0.1.2 Use full image ref for docker official images + - 0.1.3 Helm 3 - Fix Job labels + - 0.1.4 Update htk requirements + - 0.1.5 Enable taint toleration for Openstack services jobs + - 0.1.6 Update all Ceph images to Focal + - 0.1.7 Replace node-role.kubernetes.io/master with control-plane + - 0.1.8 Migrated pdb resource to policy/v1 API version + - 0.1.9 Migrated CronJob resource to batch/v1 API version + - 0.1.10 Update Ceph to 17.2.6 + - 0.1.11 Update Rook to 1.12.5 and Ceph to 18.2.0 + - 0.1.12 Update Ceph images to Jammy and Reef 18.2.1 + - 0.1.13 Bugfix Ceph user creation for RBD access + - 0.1.14 Update Ceph images to patched 18.2.2 and restore debian-reef repo + - 0.1.15 Add 2023.2 Ubuntu Jammy overrides + - 0.1.16 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.1.17 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/grafana.yaml b/releasenotes/notes/grafana.yaml new file mode 100644 index 0000000000..0ccc3f67c7 --- /dev/null +++ b/releasenotes/notes/grafana.yaml @@ -0,0 +1,36 @@ +--- +grafana: + - 0.1.0 Initial Chart + - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" + - 0.1.2 Update Grafana version + - 0.1.3 Provision any dashboard as homepage + - 0.1.4 Enable TLS for Grafana + - 0.1.5 Enable TLS between Grafana and Prometheus + - 0.1.6 Enable TLS for Grafana ingress path + - 0.1.7 Update Grafana version and Selenium script + - 0.1.8 Use full image ref for docker official images + - 0.1.9 Add Alertmanager dashboard to Grafana + - 0.1.10 Helm 3 - Fix Job labels + - 0.1.11 Update htk requirements + - 0.1.12 Add iDRAC dashboard to Grafana + - 0.1.13 Update prometheus metric name + - 0.1.14 Add run migrator job + - 0.1.15 Added OCI registry authentication + - 0.1.16 Grafana 8.5.10 with unified alerting + - 0.1.17 Fix uid for the user grafana + - 0.1.18 Migrator job is now mariadb-fail-proof + - 0.1.19 Update grafana to 9.2.10 + - 0.1.20 Upgrade osh-selenium image to latest-ubuntu_focal + - 0.1.21 Fix run migrator job deployment condition + - 0.1.22 Make selenium v4 syntax optional + - 0.1.23 Modified selenium test for compatibility + - 0.1.24 Add image rendering sidecar + - 0.1.25 Add value for rendering sidecar feature + - 0.1.26 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.1.27 Update default images tags. Add 2024.1-ubuntu_jammy overrides. + - 0.1.28 Upgrade osh-selenium image to ubuntu_jammy + - 0.1.29 Add 2024.2 overrides + - 0.1.30 Update chart helm test environment variables + - 0.1.31 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/helm-toolkit.yaml b/releasenotes/notes/helm-toolkit.yaml new file mode 100644 index 0000000000..d37a1145b3 --- /dev/null +++ b/releasenotes/notes/helm-toolkit.yaml @@ -0,0 +1,90 @@ +--- +helm-toolkit: + - 0.1.0 Initial Chart + - 0.1.1 Add extra DNS names to Ingress + - 0.1.2 Make database backups work with openstack Train + - 0.1.3 Fix ks-user script case matching for domain + - 0.1.4 Update ingress tpl in helmtoolkit + - 0.1.5 Add capability to delete a backup archive + - 0.2.0 Update default Kubernetes API for use with Helm v3 + - 0.2.1 Change Issuer to ClusterIssuer + - 0.2.2 Revert Change Issuer to ClusterIssuer + - 0.2.3 Allow openstack service list to retry in event of keystone connection issues + - 0.2.4 Added detailed FiXME for ks-service script bug and code changes + - 0.2.5 Added logic to support cert-manager versioning + - 0.2.6 Add metadata in job templates + - 0.2.7 Replace brace expansion with more standardized Posix approach + - 0.2.8 Override the expiry of Ingress TLS certificate + - 0.2.9 Jobs; put labels only in the template spec + - 0.2.10 Add more S3 configuration options + - 0.2.11 Revert S3 User & Bucket job scripts to v0.2.9 + - 0.2.12 Remove hook-delete-policy + - 0.2.13 Modify connection args for s3 bucket creation when TLS is enabled + - 0.2.14 Remove TLS_OPTION argument from s3 bucket creation job + - 0.2.15 Adding TLS rabbitmq logic + - 0.2.16 Add manual mode to the created backup file name + - 0.2.17 Update db backup/restore retry for sending to remote + - 0.2.18 Make Rabbit-init job more robust + - 0.2.19 Revoke all privileges for PUBLIC role in postgres dbs + - 0.2.20 Modify the template of rbac_role to make secrets accessible + - 0.2.21 Fix issue with db backup error return code being eaten + - 0.2.22 Add ability to set labels to add to resources + - 0.2.23 Helm 3 - Fix Job labels + - 0.2.24 Migrate Ingress resources to networking.k8s.io/v1 + - 0.2.25 Set Security Context to ks-user job + - 0.2.26 Revert Set Security Context to ks-user job + - 0.2.27 Correct private key size input for Certificates and remove minor version support + - 0.2.28 Set Security context to ks-user job at pod and container level + - 0.2.29 Enhance mariadb backup + - 0.2.30 Add ability to image pull secrets on pods + - 0.2.31 Add log strings for alert generation + - 0.2.32 Consolidate mon_endpoints discovery + - 0.2.33 Remove set -x + - 0.2.34 Modify database backup logic to maintain minimum number of backups + - 0.2.35 Database B/R improvements + - 0.2.36 Enable taint toleration for Openstack services jobs + - 0.2.37 Updated chart naming for subchart compatibility + - 0.2.38 Minor change to display archive directory with files in sub-directory + - 0.2.39 Removed tillerVersion from Chart to pass helm3 linting + - 0.2.40 Revert chart naming for subchart compatibility + - 0.2.41 Database B/R - archive name parser added + - 0.2.42 Database B/R - fix to make script compliant with a retention policy + - 0.2.43 Support having a single external ingress controller + - 0.2.44 Added OCI registry authentication + - 0.2.45 Modify use_external_ingress_controller place in openstack-helm values.yaml + - 0.2.46 Fixed for getting kibana ingress value parameters + - 0.2.47 Adjusting of kibana ingress value parameters + - 0.2.48 Added verify_databases_backup_archives function call to backup process and added remote backup sha256 hash verification + - 0.2.49 Moved RabbitMQ Guest Admin removal to init + - 0.2.50 Allow tls for external ingress without specifying key and crt + - 0.2.51 Added a random delay up to 300 seconds to remote backup upload/download for load spreading purpose + - 0.2.52 Decreased random delay to up to 30 seconds and switched remote backup verification protocol to md5 + - 0.2.53 Update create db user queries + - 0.2.54 Fix dependency resolver to ignore non-existing dependencyKey when dependencyMixinParam is a slice + - 0.2.55 Updated deprecated IngressClass annotation + - 0.2.56 Expose S3 credentials from Rook bucket CRD secret + - 0.2.57 Safer file removal + - 0.2.58 Backups verification improvements + - 0.2.59 Added throttling remote backups + - 0.2.60 Change default ingress pathType to Prefix + - 0.2.61 Add custom pod annotations snippet + - 0.2.62 Add custom secret annotations snippet + - 0.2.63 Add custom job annotations snippet and wire it into job templates + - 0.2.64 Use custom secret annotations snippet in other secret templates + - 0.2.65 Escape special characters in password for DB connection + - 0.2.66 Align db scripts with sqlalchemy 2.0 + - 0.2.67 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.2.68 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.2.69 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.2.70 Decode url-encoded password for rabbit connection + - 0.2.71 Add snippet with service parameters + - 0.2.72 Add snippet configmap_oslo_policy + - 0.2.73 Add ability to get multiple hosts endpoint + - 0.2.74 Remove trailing slash in endpoinds + - 0.2.75 Add daemonset_overrides_root util + - 0.2.76 update tookit to support fqdn alias + - 0.2.77 Add recommended kubernetes name label to pods definition + - 0.2.78 Fix db-init and db-drop scripts to make them work with sqlalchemy >2.0 + - 0.2.79 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/ingress.yaml b/releasenotes/notes/ingress.yaml new file mode 100644 index 0000000000..b579cd53a6 --- /dev/null +++ b/releasenotes/notes/ingress.yaml @@ -0,0 +1,26 @@ +--- +ingress: + - 0.1.0 Initial Chart + - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" + - 0.1.2 Update to container image repo k8s.gcr.io + - 0.2.0 Update default Kubernetes API for use with Helm v3 + - 0.2.1 Use HostToContainer mountPropagation + - 0.2.2 Use full image ref for docker official images + - 0.2.3 Uplift ingress to 0.42.0 + - 0.2.4 Update htk requirements + - 0.2.5 Migrate Ingress resources to networking.k8s.io/v1 + - 0.2.6 Add option to assign VIP as externalIP + - 0.2.7 Enable taint toleration for Openstack services jobs + - 0.2.8 Uplift ingress to 1.1.3 + - 0.2.9 Added OCI registry authentication + - 0.2.10 Update neutron images to xena release + - 0.2.11 Fix resource name in the role + - 0.2.12 Uplift ingress to 1.5.1 + - 0.2.13 Allow setting node_port for the svc + - 0.2.14 Replace node-role.kubernetes.io/master with control-plane + - 0.2.15 Update kubernetes registry to registry.k8s.io + - 0.2.16 Updated deprecated IngressClass annotation + - 0.2.17 Fixed controller parameters + - 0.2.18 Fixed some additional controller issues + - 0.2.19 Uplift ingress controller image to 1.8.2 +... diff --git a/releasenotes/notes/kibana.yaml b/releasenotes/notes/kibana.yaml new file mode 100644 index 0000000000..7d982031cd --- /dev/null +++ b/releasenotes/notes/kibana.yaml @@ -0,0 +1,25 @@ +--- +kibana: + - 0.1.0 Initial Chart + - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" + - 0.1.2 Drop usage of fsGroup inside container + - 0.1.3 Enable TLS with Elasticsearch + - 0.1.4 Enable TLS for Kibana ingress path + - 0.1.5 Use full image ref for docker official images + - 0.1.6 Remove Kibana indices before pod start up + - 0.1.7 Helm 3 - Fix Job labels + - 0.1.8 Update htk requirements + - 0.1.9 Revert removing Kibana indices before pod start up + - 0.1.10 Update image defaults + - 0.1.11 Added OCI registry authentication + - 0.1.12 Added feedback http_code 200 for kibana indexes + - 0.1.13 Update Kibana to 8.9.0 + - 0.1.14 Add 2023.1 Ubuntu Focal overrides + - 0.1.15 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.1.16 Add 2024.1 Ubuntu Jammy overrides + - 0.1.17 Update script to use data views replacing deprecated api + - 0.1.18 Add retry logic to create_kibana_index_patterns.sh + - 0.1.19 Add 2024.2 overrides + - 0.1.20 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/kube-dns.yaml b/releasenotes/notes/kube-dns.yaml new file mode 100644 index 0000000000..b98bdbc80b --- /dev/null +++ b/releasenotes/notes/kube-dns.yaml @@ -0,0 +1,14 @@ +--- +kube-dns: + - 0.1.0 Initial Chart + - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" + - 0.1.2 Update to container image repo k8s.gcr.io + - 0.1.3 Use full image ref for docker official images + - 0.1.4 Update htk requirements + - 0.1.5 Added OCI registry authentication + - 0.1.6 Replace node-role.kubernetes.io/master with control-plane + - 0.1.7 Update kubernetes registry to registry.k8s.io + - 0.1.8 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.1.9 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/kubernetes-keystone-webhook.yaml b/releasenotes/notes/kubernetes-keystone-webhook.yaml new file mode 100644 index 0000000000..63da32cfd1 --- /dev/null +++ b/releasenotes/notes/kubernetes-keystone-webhook.yaml @@ -0,0 +1,17 @@ +--- +kubernetes-keystone-webhook: + - 0.1.0 Initial Chart + - 0.1.1 Update k8s-keystone-auth version + - 0.1.2 Change helm-toolkit dependency version to ">= 0.1.0" + - 0.1.3 Remove Kibana source reference + - 0.1.4 Use full image ref for docker official images + - 0.1.5 Update htk requirements + - 0.1.6 Update default image value to Wallaby + - 0.1.7 Added OCI registry authentication + - 0.1.8 Add 2023.1 Ubuntu Focal overrides + - 0.1.9 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.1.10 Add 2024.1 Ubuntu Jammy overrides + - 0.1.11 Add 2024.2 overrides + - 0.1.12 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/kubernetes-node-problem-detector.yaml b/releasenotes/notes/kubernetes-node-problem-detector.yaml new file mode 100644 index 0000000000..b66277ba60 --- /dev/null +++ b/releasenotes/notes/kubernetes-node-problem-detector.yaml @@ -0,0 +1,16 @@ +--- +kubernetes-node-problem-detector: + - 0.1.0 Initial Chart + - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" + - 0.1.2 Unpin images built with osh-images + - 0.1.3 Update RBAC apiVersion from /v1beta1 to /v1 + - 0.1.4 Update the systemd-monitor lookback duration + - 0.1.5 Use full image ref for docker official images + - 0.1.6 Update htk requirements + - 0.1.7 Added OCI registry authentication + - 0.1.8 Replace node-role.kubernetes.io/master with control-plane + - 0.1.9 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.1.10 Update node_problem_detector to latest-ubuntu_jammy + - 0.1.11 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/ldap.yaml b/releasenotes/notes/ldap.yaml new file mode 100644 index 0000000000..7a0a2c1b87 --- /dev/null +++ b/releasenotes/notes/ldap.yaml @@ -0,0 +1,11 @@ +--- +ldap: + - 0.1.0 Initial Chart + - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" + - 0.1.2 Use full image ref for docker official images + - 0.1.3 Update htk requirements + - 0.1.4 Added OCI registry authentication + - 0.1.5 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.1.6 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/libvirt.yaml b/releasenotes/notes/libvirt.yaml new file mode 100644 index 0000000000..1634d50365 --- /dev/null +++ b/releasenotes/notes/libvirt.yaml @@ -0,0 +1,46 @@ +--- +libvirt: + - 0.1.0 Initial Chart + - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" + - 0.1.2 Setup libvirt SSL + - 0.1.3 Create override for external ceph cinder backend + - 0.1.4 Set unix socket auth method as none + - 0.1.5 Use full image ref for docker official images + - 0.1.6 Enhancement to enable probes override from values.yaml + - 0.1.7 Add libvirt overrides for Victoria and Wallaby + - 0.1.8 Update htk requirements + - 0.1.9 Exec libvirt instead of forking from bash + - 0.1.10 Enable taint toleration for Openstack services jobs + - 0.1.11 Remove unused overrides and update default image + - 0.1.12 Add libvirt exporter as a sidecar + - 0.1.13 Added OCI registry authentication + - 0.1.14 Remove use of exec in libvirt.sh + - 0.1.15 Add support for libvirt to connect to external ceph without any local ceph present + - 0.1.16 Update all Ceph images to Focal + - 0.1.17 Add ovn.yaml values_override, remove dependency from neutron-ovs-agent module + - 0.1.18 Replace node-role.kubernetes.io/master with control-plane + - 0.1.19 Set kubernetes cgroup value equal kubepods.slice to fit systemd cgroup driver + - 0.1.20 Update Ceph to 17.2.6 + - 0.1.21 Disable libvirt cgroup functionality for cgroup-v2 + - 0.1.22 Set targeted dependency of libvirt with ovn networking backend + - 0.1.23 Add support for enabling vencrypt + - 0.1.24 Include HOSTNAME_FQDN for certificates + - 0.1.25 Add 2023.2 Ubuntu Jammy overrides + - 0.1.26 Update Rook to 1.12.5 and Ceph to 18.2.0 + - 0.1.27 Add watch verb to vencrypt cert-manager Role + - 0.1.28 Update Ceph images to Jammy and Reef 18.2.1 + - 0.1.29 Update Ceph images to patched 18.2.2 and restore debian-reef repo + - 0.1.30 Add 2024.1 overrides + - 0.1.31 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.1.32 Enable a flag to parse Libvirt Nova metadata in libvirt exporter + - 0.1.33 Handle cgroupv2 correctly + - 0.1.34 Remove hugepages creation test + - 0.1.35 Allow to initialize virtualization modules + - 0.1.36 Allow to generate dynamic config options + - 0.1.37 Make readiness probes more tiny + - 0.1.38 Implement daemonset overrides for libvirt + - 0.1.39 Add 2023.1 overrides for Ubuntu Focal and Jammy + - 0.1.40 Add 2024.2 overrides + - 0.1.41 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/local-storage.yaml b/releasenotes/notes/local-storage.yaml new file mode 100644 index 0000000000..90ca799b40 --- /dev/null +++ b/releasenotes/notes/local-storage.yaml @@ -0,0 +1,8 @@ +--- +local-storage: + - 0.1.0 Initial Chart + - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" + - 0.1.2 Update htk requirements + - 0.1.3 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/local-volume-provisioner.yaml b/releasenotes/notes/local-volume-provisioner.yaml new file mode 100644 index 0000000000..acdb52f2ab --- /dev/null +++ b/releasenotes/notes/local-volume-provisioner.yaml @@ -0,0 +1,6 @@ +--- +local-volume-provisioner: + - 0.1.0 Initial Chart + - 0.1.1 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/lockdown.yaml b/releasenotes/notes/lockdown.yaml new file mode 100644 index 0000000000..4ad8013b7f --- /dev/null +++ b/releasenotes/notes/lockdown.yaml @@ -0,0 +1,7 @@ +--- +lockdown: + - 0.1.0 Initial Chart + - 0.1.1 Allows toggling + - 0.1.2 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/mariadb-backup.yaml b/releasenotes/notes/mariadb-backup.yaml new file mode 100644 index 0000000000..005a2f5661 --- /dev/null +++ b/releasenotes/notes/mariadb-backup.yaml @@ -0,0 +1,12 @@ +--- +mariadb-backup: + - 0.0.1 Initial Chart + - 0.0.2 Added staggered backups support + - 0.0.3 Backups verification improvements + - 0.0.4 Added throttling remote backups + - 0.0.5 Add 2024.1 overrides + - 0.0.6 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.0.7 Add 2024.2 overrides + - 0.0.8 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/mariadb-cluster.yaml b/releasenotes/notes/mariadb-cluster.yaml new file mode 100644 index 0000000000..b126dba929 --- /dev/null +++ b/releasenotes/notes/mariadb-cluster.yaml @@ -0,0 +1,12 @@ +--- +mariadb-cluster: + - 0.0.1 Initial Chart + - 0.0.2 Enable auto-upgrade + - 0.0.3 Fixed TLS config and added x509 requirement + - 0.0.4 Add 2024.1 overrides + - 0.0.5 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.0.6 Add 2024.2 overrides + - 0.0.7 Allow to use default storage class + - 0.0.8 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/mariadb.yaml b/releasenotes/notes/mariadb.yaml new file mode 100644 index 0000000000..9f651a5abf --- /dev/null +++ b/releasenotes/notes/mariadb.yaml @@ -0,0 +1,87 @@ +--- +mariadb: + - 0.1.0 Initial Chart + - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" + - 0.1.2 mariadb security best practice fixes + - 0.1.3 Fix MariaDB backup script + - 0.1.4 Unpin images built with osh-images + - 0.1.5 Update to container image repo k8s.gcr.io + - 0.1.6 Change Issuer to ClusterIssuer + - 0.1.7 Revert - Change Issuer to ClusterIssuer + - 0.1.8 Change Issuer to ClusterIssuer with logic in place to support cert-manager versioning + - 0.1.9 Uplift Mariadb-ingress to 0.42.0 + - 0.1.10 Rename mariadb backup identities + - 0.1.11 Disable mariadb mysql history client logging + - 0.1.12 Set strict permission on mariadb data dir + - 0.1.13 Fix race condition for grastate.dat + - 0.1.14 Update mysqld-exporter image to v0.12.1 + - 0.2.0 Uplift mariadb version and ubuntu release + - 0.2.1 Prevent potential splitbrain issue if cluster is in reboot state + - 0.2.2 remove deprecated svc annotation tolerate-unready-endpoints + - 0.2.3 Remove panko residue + - 0.2.4 Use full image ref for docker official images + - 0.2.5 Added helm hook for post-install and post-upgrade in prometheus exporter job. + - 0.2.6 Update log format stream for mariadb + - 0.2.7 add ingress resources + - 0.2.8 Helm 3 - Fix Job labels + - 0.2.9 Update htk requirements + - 0.2.10 Fix Python exceptions + - 0.2.11 Enhance mariadb backup + - 0.2.12 Remove set -x + - 0.2.13 Adjust readiness.sh in single node and no replication case + - 0.2.14 Fix comparison value + - 0.2.15 Updated naming for subchart compatibility + - 0.2.16 Revert naming for subchart compatibility + - 0.2.17 Enable taint toleration for Openstack services jobs + - 0.2.18 Updated naming for subchart compatibility + - 0.2.19 Update default image value to Wallaby + - 0.2.20 Migrated CronJob resource to batch/v1 API version & PodDisruptionBudget to policy/v1; Uplift Mariadb-ingress to 1.1.3 + - 0.2.21 Fix mysql exporter user privileges + - 0.2.22 Fix ingress cluster role privileges + - 0.2.23 Fix backup script by ignoring sys database for MariaDB 10.6 compartibility + - 0.2.24 Uplift Mariadb-ingress to 1.2.0 + - 0.2.25 Add liveness probe to restart a pod that got stuck in a transfer wsrep_local_state_comment + - 0.2.26 Added OCI registry authentication + - 0.2.27 Fix broken helmrelease for helmv3 + - 0.2.28 Added verify_databases_backup_in_directory function implementation + - 0.2.29 Uplift Mariadb-ingress to 1.5.1 + - 0.2.30 Replace node-role.kubernetes.io/master with control-plane + - 0.2.31 Update kubernetes registry to registry.k8s.io + - 0.2.32 Prevent liveness probe from killing pods during SST + - 0.2.33 Add 2023.1 Ubuntu Focal overrides + - 0.2.34 Uplift ingress controller image to 1.8.2 + - 0.2.35 Update apparmor override + - 0.2.36 Added staggered backups support + - 0.2.37 Backups verification improvements + - 0.2.38 Added throttling remote backups + - 0.2.39 Template changes for image 1.9 compatibility + - 0.2.40 Start.py allows to create mariadb-service-primary service and endpoint + - 0.2.41 Switch to primary service instead of ingress by default + - 0.2.42 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.2.43 Add 2024.1 Ubuntu Jammy overrides + - 0.2.44 Uplift ingress controller image to 1.11.2 + - 0.2.45 Add mariadb controller support + - 0.2.46 Avoid using cluster endpoints + - 0.2.47 Deploy exporter as sidecar + - 0.2.48 Switch to mariadb controller deployment + - 0.2.49 Remove ingress deployment + - 0.2.50 Add cluster-wait job + - 0.2.51 Add 2024.2 overrides + - 0.2.52 Added SSL support to cluster-wait job + - 0.2.53 Use constant for mysql binary name + - 0.2.54 Improve leader election on cold start + - 0.2.55 Improve python3 compatibility + - 0.2.56 Stop running threads on sigkill + - 0.2.57 Remove useless retries on conflicts during cm update + - 0.2.58 Prevent TypeError in get_active_endpoint function + - 0.2.59 Give more time on resolving configmap update conflicts + - 0.2.60 Refactor liveness/readiness probes + - 0.2.61 Avoid using deprecated isAlive() + - 0.2.62 Implement mariadb upgrade during start + - 0.2.63 Use service ip for endpoint discovery + - 0.2.64 Add terminationGracePeriodSeconds + - 0.2.65 Allow to use default storage class + - 0.2.66 Add probes for exporter + - 0.2.67 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/memcached.yaml b/releasenotes/notes/memcached.yaml new file mode 100644 index 0000000000..a51e11863e --- /dev/null +++ b/releasenotes/notes/memcached.yaml @@ -0,0 +1,23 @@ +--- +memcached: + - 0.1.0 Initial Chart + - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" + - 0.1.2 Make stats cachedump configurable. + - 0.1.3 Remove panko residue + - 0.1.4 Use full image ref for docker official images + - 0.1.5 Update htk requirements + - 0.1.6 Switch to using sidecar for exporter + - 0.1.7 Updated naming for subchart compatibility + - 0.1.8 Enable taint toleration for Openstack services jobs + - 0.1.9 Revert naming for subchart compatibility + - 0.1.10 Updated naming for subchart compatibility + - 0.1.11 Remove gnocchi netpol override + - 0.1.12 Added OCI registry authentication + - 0.1.13 Replace node-role.kubernetes.io/master with control-plane + - 0.1.14 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.1.15 Allow to pass additional service parameters + - 0.1.16 Change deployment type to statefulset + - 0.1.17 Fix statefulset spec format + - 0.1.18 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/metacontroller.yaml b/releasenotes/notes/metacontroller.yaml new file mode 100644 index 0000000000..a09e3ba3df --- /dev/null +++ b/releasenotes/notes/metacontroller.yaml @@ -0,0 +1,13 @@ +--- +metacontroller: + - 0.1.0 Initial Chart + - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" + - 0.1.2 Fix disappearing metacontroller CRDs on upgrade + - 0.1.3 Use full image ref for docker official images + - 0.1.4 Update htk requirements + - 0.1.5 Fix field validation error + - 0.1.6 Added OCI registry authentication + - 0.1.7 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.1.8 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/mongodb.yaml b/releasenotes/notes/mongodb.yaml new file mode 100644 index 0000000000..cebe505efc --- /dev/null +++ b/releasenotes/notes/mongodb.yaml @@ -0,0 +1,12 @@ +--- +mongodb: + - 0.1.0 Initial Chart + - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" + - 0.1.2 Use full image ref for docker official images + - 0.1.3 Update htk requirements + - 0.1.4 Added OCI registry authentication + - 0.1.5 Add conf file for MongoDB + - 0.1.6 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.1.7 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/nagios.yaml b/releasenotes/notes/nagios.yaml new file mode 100644 index 0000000000..e36c60c1db --- /dev/null +++ b/releasenotes/notes/nagios.yaml @@ -0,0 +1,20 @@ +--- +nagios: + - 0.1.0 Initial Chart + - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" + - 0.1.2 Use full image ref for docker official images + - 0.1.3 Mount internal TLS CA certificate + - 0.1.4 Update htk requirements + - 0.1.5 Switch nagios image from xenial to bionic + - 0.1.6 Added OCI registry authentication + - 0.1.7 Upgrade osh-selenium image to latest-ubuntu_focal + - 0.1.8 Use helm toolkit for readiness probes + - 0.1.9 Make using selenium v4 syntax optional + - 0.1.10 Correct selenium v3 syntax + - 0.1.11 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.1.12 Update nagios image tag to latest-ubuntu_jammy + - 0.1.13 Add the ability to use custom Nagios plugins + - 0.1.14 Upgrade osh-selenium image to ubuntu_jammy + - 0.1.15 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/namespace-config.yaml b/releasenotes/notes/namespace-config.yaml new file mode 100644 index 0000000000..9243e089a6 --- /dev/null +++ b/releasenotes/notes/namespace-config.yaml @@ -0,0 +1,8 @@ +--- +namespace-config: + - 0.1.0 Initial Chart + - 0.1.1 Grant access to existing PodSecurityPolicy + - 0.1.2 Rmove PodSecurityPolicy + - 0.1.3 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/nfs-provisioner.yaml b/releasenotes/notes/nfs-provisioner.yaml new file mode 100644 index 0000000000..ac21181ae4 --- /dev/null +++ b/releasenotes/notes/nfs-provisioner.yaml @@ -0,0 +1,12 @@ +--- +nfs-provisioner: + - 0.1.0 Initial Chart + - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" + - 0.1.2 Use full image ref for docker official images + - 0.1.3 Update htk requirements + - 0.1.4 Added OCI registry authentication + - 0.1.5 Update image version + - 0.1.6 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.1.7 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/openvswitch-0b37403ffc75bb63.yaml b/releasenotes/notes/openvswitch-0b37403ffc75bb63.yaml new file mode 100644 index 0000000000..89dfd12921 --- /dev/null +++ b/releasenotes/notes/openvswitch-0b37403ffc75bb63.yaml @@ -0,0 +1,4 @@ +--- +openvswitch: + - Change Open vSwitch to run with non-root user +... diff --git a/releasenotes/notes/openvswitch-5c0d74ca4f420e56.yaml b/releasenotes/notes/openvswitch-5c0d74ca4f420e56.yaml new file mode 100644 index 0000000000..fdd62d4a1e --- /dev/null +++ b/releasenotes/notes/openvswitch-5c0d74ca4f420e56.yaml @@ -0,0 +1,4 @@ +--- +openvswitch: + - Set nova user as owner for hugepages mount path +... diff --git a/releasenotes/notes/openvswitch-e761d6733b84bdc7.yaml b/releasenotes/notes/openvswitch-e761d6733b84bdc7.yaml new file mode 100644 index 0000000000..e818af28cc --- /dev/null +++ b/releasenotes/notes/openvswitch-e761d6733b84bdc7.yaml @@ -0,0 +1,4 @@ +--- +openvswitch: + - Make the --user flag for OVS server optional +... diff --git a/releasenotes/notes/openvswitch.yaml b/releasenotes/notes/openvswitch.yaml new file mode 100644 index 0000000000..56d077c9e7 --- /dev/null +++ b/releasenotes/notes/openvswitch.yaml @@ -0,0 +1,31 @@ +--- +openvswitch: + - 0.1.0 Initial Chart + - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" + - 0.1.2 Unpin images built with osh-images + - 0.1.3 Use HostToContainer mountPropagation + - 0.1.4 Support override of vswitchd liveness and readiness probe + - 0.1.5 Use full image ref for docker official images + - 0.1.6 Update htk requirements + - 0.1.7 Enable taint toleration for Openstack services jobs + - 0.1.8 Added OCI registry authentication + - 0.1.9 Enable ovs hardware offload + - 0.1.10 Merge ovs-db and ovs-vswitchd in one Daemonset + - 0.1.11 Add ovn.yaml in values_override, Enable ptcp_port 6640 which needed when use ovn + - 0.1.12 Replace node-role.kubernetes.io/master with control-plane + - 0.1.13 Upgrade openvswitch image to latest-ubuntu_focal to fix qos issue + - 0.1.14 Add buffer before accesses pid file + - 0.1.15 Add buffer before accesses ovs controller pid socket + - 0.1.16 Restore ServiceAccount to openvswitch pod + - 0.1.17 Add buffer to wait for potential new CTL file before running chown + - 0.1.18 Add value for extra poststart command + - 0.1.19 Add check for cgroups v2 file structure + - 0.1.20 Add Ubuntu Focal and Ubuntu Jammy overrides + - 0.1.21 Add overrides for dpdk + - 0.1.22 Change hugepages size to 2M for easier configuration + - 0.1.23 Fix rolebinding for init container + - 0.1.24 Change ovs to run as child process of start script + - 0.1.25 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.1.26 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/ovn-3b9e82e5d469bc98.yaml b/releasenotes/notes/ovn-3b9e82e5d469bc98.yaml new file mode 100644 index 0000000000..454492bf74 --- /dev/null +++ b/releasenotes/notes/ovn-3b9e82e5d469bc98.yaml @@ -0,0 +1,4 @@ +--- +features: + - Implement daemonset overrides +... diff --git a/releasenotes/notes/ovn-50ba6d3611decff9.yaml b/releasenotes/notes/ovn-50ba6d3611decff9.yaml new file mode 100644 index 0000000000..f71d1ec9f9 --- /dev/null +++ b/releasenotes/notes/ovn-50ba6d3611decff9.yaml @@ -0,0 +1,4 @@ +--- +ovn: + - Add OVN Kubernetes support +... diff --git a/releasenotes/notes/ovn-a82eced671495a3d.yaml b/releasenotes/notes/ovn-a82eced671495a3d.yaml new file mode 100644 index 0000000000..c429489654 --- /dev/null +++ b/releasenotes/notes/ovn-a82eced671495a3d.yaml @@ -0,0 +1,4 @@ +--- +ovn: + - Add OVN network logging parser +... diff --git a/releasenotes/notes/ovn.yaml b/releasenotes/notes/ovn.yaml new file mode 100644 index 0000000000..7d75c76048 --- /dev/null +++ b/releasenotes/notes/ovn.yaml @@ -0,0 +1,21 @@ +--- +ovn: + - 0.1.0 Add OVN! + - 0.1.1 Fix ovn db persistence issue + - 0.1.2 Add bridge-mapping configuration + - 0.1.3 Fix system-id reuse + - 0.1.4 Add support for OVN HA + refactor + - 0.1.5 Add ubuntu_focal and ubuntu_jammy overrides + - 0.1.6 Fix ovsdb port number + - 0.1.7 Use host network for ovn controller pods + - 0.1.8 Fix attaching interfaces to the bridge + - 0.1.9 Make ovn db file path as configurable + - 0.1.10 Fix typo in the controller init script + - 0.1.11 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.1.12 Fix oci_image_registry secret name + - 0.1.13 Allow share OVN DB NB/SB socket + - 0.1.14 Make the label for OVN controller gateway configurable + - 0.1.15 Fix resources + - 0.1.16 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/podsecuritypolicy.yaml b/releasenotes/notes/podsecuritypolicy.yaml new file mode 100644 index 0000000000..a4b083c656 --- /dev/null +++ b/releasenotes/notes/podsecuritypolicy.yaml @@ -0,0 +1,7 @@ +--- +podsecuritypolicy: + - 0.1.0 Initial Chart + - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" + - 0.1.2 Update htk requirements + - 1.0.0 Remove chart due to PodSecurityPolicy deprecation +... diff --git a/releasenotes/notes/postgresql.yaml b/releasenotes/notes/postgresql.yaml new file mode 100644 index 0000000000..507c3c73de --- /dev/null +++ b/releasenotes/notes/postgresql.yaml @@ -0,0 +1,29 @@ +--- +postgresql: + - 0.1.0 Initial Chart + - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" + - 0.1.2 adding archiving to postgres + - 0.1.3 Use explicit entrypoint for prometheus exporter + - 0.1.4 Allow probe tweaking + - 0.1.5 Optimize restart behavior + - 0.1.6 Revert "Add default reject rule ..." + - 0.1.7 postgres archive cleanup script + - 0.1.8 Add tls to Postgresql + - 0.1.9 Use full image ref for docker official images + - 0.1.10 Helm 3 - Fix Job labels + - 0.1.11 Update htk requirements + - 0.1.12 Enhance postgresql backup + - 0.1.13 Remove set -x + - 0.1.14 Fix invalid fields in values + - 0.1.15 Migrated CronJob resource to batch/v1 API version + - 0.1.16 Added OCI registry authentication + - 0.1.17 Added empty verify_databases_backup_archives() function implementation to match updated backup_databases() function in helm-toolkit + - 0.1.18 Updated postgres to 14.5 and replaced deprecated config item wal_keep_segments with wal_keep_size + - 0.1.19 Added staggered backups support + - 0.1.20 Added throttling remote backups + - 0.1.21 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.1.22 Update default images tags. Add 2024.1-ubuntu_jammy overrides. + - 0.1.23 Add 2024.2 overrides + - 0.1.24 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/powerdns.yaml b/releasenotes/notes/powerdns.yaml new file mode 100644 index 0000000000..90e9208cac --- /dev/null +++ b/releasenotes/notes/powerdns.yaml @@ -0,0 +1,16 @@ +--- +powerdns: + - 0.1.0 Initial Chart + - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" + - 0.1.2 Use full image ref for docker official images + - 0.1.3 Helm 3 - Fix Job labels + - 0.1.4 Update htk requirements + - 0.1.5 Update default image values + - 0.1.6 Added OCI registry authentication + - 0.1.7 Add 2023.1 Ubuntu Focal overrides + - 0.1.8 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.1.9 Add 2024.1 Ubuntu Jammy overrides + - 0.1.10 Add 2024.2 overrides + - 0.1.11 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/prometheus-alertmanager.yaml b/releasenotes/notes/prometheus-alertmanager.yaml new file mode 100644 index 0000000000..ccbe06450e --- /dev/null +++ b/releasenotes/notes/prometheus-alertmanager.yaml @@ -0,0 +1,16 @@ +--- +prometheus-alertmanager: + - 0.1.0 Initial Chart + - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" + - 0.1.2 Add extensible command line flags to Alertmanager + - 0.1.3 Add LDAP to Alertmanager + - 0.1.4 Remove snmp_notifier subchart from alertmanager + - 0.1.5 Add Prometheus Scrape Annotation + - 0.1.6 Remove Alerta from openstack-helm-infra repository + - 0.1.7 Use full image ref for docker official images + - 0.1.8 Update htk requirements + - 0.1.9 Added OCI registry authentication + - 0.1.10 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.1.11 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/prometheus-blackbox-exporter.yaml b/releasenotes/notes/prometheus-blackbox-exporter.yaml new file mode 100644 index 0000000000..82143f41f1 --- /dev/null +++ b/releasenotes/notes/prometheus-blackbox-exporter.yaml @@ -0,0 +1,11 @@ +--- +prometheus-blackbox-exporter: + - 0.1.0 Initial Chart + - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" + - 0.1.2 Rename image key name + - 0.1.3 Update htk requirements + - 0.1.4 Fix indentation + - 0.1.5 Added OCI registry authentication + - 0.1.6 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/prometheus-kube-state-metrics.yaml b/releasenotes/notes/prometheus-kube-state-metrics.yaml new file mode 100644 index 0000000000..8146a32bfa --- /dev/null +++ b/releasenotes/notes/prometheus-kube-state-metrics.yaml @@ -0,0 +1,14 @@ +--- +prometheus-kube-state-metrics: + - 0.1.0 Initial Chart + - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" + - 0.1.2 Update to make current + - 0.1.3 Update image version from v2.0.0-alpha to v2.0.0-alpha-1 + - 0.1.4 Use full image ref for docker official images + - 0.1.5 Fix helm3 compatability + - 0.1.6 Update htk requirements + - 0.1.7 Added OCI registry authentication + - 0.1.8 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.1.9 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/prometheus-mysql-exporter.yaml b/releasenotes/notes/prometheus-mysql-exporter.yaml new file mode 100644 index 0000000000..c54dfa5d16 --- /dev/null +++ b/releasenotes/notes/prometheus-mysql-exporter.yaml @@ -0,0 +1,10 @@ +--- +prometheus-mysql-exporter: + - 0.0.1 Initial Chart + - 0.0.2 Add 2024.1 overrides + - 0.0.3 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.0.4 Fix typo in the values_overrides directory name + - 0.0.5 Add 2024.2 overrides + - 0.0.6 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/prometheus-node-exporter.yaml b/releasenotes/notes/prometheus-node-exporter.yaml new file mode 100644 index 0000000000..1f1389bf9a --- /dev/null +++ b/releasenotes/notes/prometheus-node-exporter.yaml @@ -0,0 +1,13 @@ +--- +prometheus-node-exporter: + - 0.1.0 Initial Chart + - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" + - 0.1.2 Add possibility to use overrides for some charts + - 0.1.3 Use full image ref for docker official images + - 0.1.4 Update htk requirements + - 0.1.5 Added OCI registry authentication + - 0.1.6 Replace node-role.kubernetes.io/master with control-plane + - 0.1.7 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.1.8 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/prometheus-openstack-exporter.yaml b/releasenotes/notes/prometheus-openstack-exporter.yaml new file mode 100644 index 0000000000..c7f10d46c2 --- /dev/null +++ b/releasenotes/notes/prometheus-openstack-exporter.yaml @@ -0,0 +1,15 @@ +--- +prometheus-openstack-exporter: + - 0.1.0 Initial Chart + - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" + - 0.1.2 Unpin prometheus-openstack-exporter image + - 0.1.3 Add possibility to use overrides for some charts + - 0.1.4 Use full image ref for docker official images + - 0.1.5 Helm 3 - Fix Job labels + - 0.1.6 Update htk requirements + - 0.1.7 Added OCI registry authentication + - 0.1.8 Switch to jammy-based images + - 0.1.9 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.1.10 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/prometheus-process-exporter.yaml b/releasenotes/notes/prometheus-process-exporter.yaml new file mode 100644 index 0000000000..f0e19ca4a3 --- /dev/null +++ b/releasenotes/notes/prometheus-process-exporter.yaml @@ -0,0 +1,13 @@ +--- +prometheus-process-exporter: + - 0.1.0 Initial Chart + - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" + - 0.1.2 Fix values_overrides directory naming + - 0.1.3 Use full image ref for docker official images + - 0.1.4 Update htk requirements + - 0.1.5 Added OCI registry authentication + - 0.1.6 Replace node-role.kubernetes.io/master with control-plane + - 0.1.7 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.1.8 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/prometheus.yaml b/releasenotes/notes/prometheus.yaml new file mode 100644 index 0000000000..ebf9780369 --- /dev/null +++ b/releasenotes/notes/prometheus.yaml @@ -0,0 +1,24 @@ +--- +prometheus: + - 0.1.0 Initial Chart + - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" + - 0.1.2 Add configurable readiness/liveness Probes + - 0.1.3 Revert "Render Rules as Templates" + - 0.1.4 Fix spacing inconsistencies with flags + - 0.1.5 Fix spacing inconsistencies with flags + - 0.1.6 Upgrade version to v2.25 fix/remove deprecated flags + - 0.1.7 Enable TLS for Prometheus + - 0.1.8 Change readiness probe from /status to /-/ready + - 0.1.9 Retrieve backend port name from values.yaml + - 0.1.10 Use full image ref for docker official images + - 0.1.11 Update htk requirements + - 0.1.12 Update default image value to Wallaby + - 0.1.13 Added OCI registry authentication + - 0.1.14 Added feature to launch Prometheus with custom script + - 0.1.15 Add 2023.1 Ubuntu Focal overrides + - 0.1.16 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.1.17 Add 2024.1 Ubuntu Jammy overrides + - 0.1.18 Add 2024.2 overrides + - 0.1.19 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/rabbitmq.yaml b/releasenotes/notes/rabbitmq.yaml new file mode 100644 index 0000000000..7177c91ce9 --- /dev/null +++ b/releasenotes/notes/rabbitmq.yaml @@ -0,0 +1,49 @@ +--- +rabbitmq: + - 0.1.0 Initial Chart + - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" + - 0.1.2 changes rmq-exporter secret src + - 0.1.4 Add configurable RABBIT_TIMEOUT parameter + - 0.1.5 Update Rabbitmq exporter version + - 0.1.6 Disallow privilege escalation in rabbitmq server container + - 0.1.7 Adding TLS logic to rabbitmq + - 0.1.8 Make helm test work with TLS + - 0.1.9 Use full image ref for docker official images + - 0.1.10 Set separate for HTTPS + - 0.1.11 Add TLS support for helm test + - 0.1.12 Added helm hook post-install and post-upgrade for rabbitmq wait cluster job + - 0.1.13 Add prestop action and version 3.8.x upgrade prep + - 0.1.14 Update readiness and liveness probes + - 0.1.15 Update htk requirements + - 0.1.16 Add force_boot command to rabbit start template + - 0.1.17 Updated naming for subchart compatibility + - 0.1.18 Revert naming for subchart compatibility + - 0.1.19 Enable taint toleration for Openstack services jobs + - 0.1.20 Bump Rabbitmq version to 3.9.0 + - 0.1.21 Updated naming for subchart compatibility + - 0.1.22 Remove guest admin account + - 0.1.23 Fixed guest account removal + - 0.1.24 Added OCI registry authentication + - 0.1.25 Add hostPort support + - 0.1.26 Moved guest admin removal to init template + - 0.1.27 Replace node-role.kubernetes.io/master with control-plane + - 0.1.28 Add IPv6 environment support for rabbitmq + - 0.1.29 Add build-in prometheus plugin and disable external exporter + - 0.1.30 Add labels to rabbitmq service + - 0.1.31 Support management api metrics collection + - 0.1.32 Enable addition of default consumer prefetch count + - 0.1.33 Bump RabbitMQ image version to 3.13.0 + - 0.1.34 Add 2024.1 overrides + - 0.1.35 Add configurable probes to rabbitmq container + - 0.1.36 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.1.37 Update rabbitmq readiness/liveness command + - 0.1.38 Do not use hardcoded username in rabbitmq chown container + - 0.1.39 Allow to bootstrap rabbitmq with initial config + - 0.1.40 Set password for guest user rabbitmq + - 0.1.41 Use short rabbitmq node name + - 0.1.42 Revert Use short rabbitmq node name + - 0.1.43 Add 2024.2 overrides + - 0.1.44 Allow to use default storage class + - 0.1.45 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/redis.yaml b/releasenotes/notes/redis.yaml new file mode 100644 index 0000000000..ebb5170d38 --- /dev/null +++ b/releasenotes/notes/redis.yaml @@ -0,0 +1,11 @@ +--- +redis: + - 0.1.0 Initial Chart + - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" + - 0.1.2 Use full image ref for docker official images + - 0.1.3 Update htk requirements + - 0.1.4 Added OCI registry authentication + - 0.1.5 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.1.6 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/registry.yaml b/releasenotes/notes/registry.yaml new file mode 100644 index 0000000000..e9e3e2bd46 --- /dev/null +++ b/releasenotes/notes/registry.yaml @@ -0,0 +1,16 @@ +--- +registry: + - 0.1.0 Initial Chart + - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" + - 0.1.2 Update to container image repo k8s.gcr.io + - 0.1.3 Use full image ref for docker official images + - 0.1.4 Helm 3 - Fix Job labels + - 0.1.5 Update htk requirements + - 0.1.6 Added OCI registry authentication + - 0.1.7 Update kubernetes registry to registry.k8s.io + - 0.1.8 Update bootstrap image url for newer image format + - 0.1.9 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.1.10 Allow to use default storage class + - 0.1.11 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/releasenotes/notes/shaker.yaml b/releasenotes/notes/shaker.yaml new file mode 100644 index 0000000000..b13b9d39df --- /dev/null +++ b/releasenotes/notes/shaker.yaml @@ -0,0 +1,13 @@ +--- +shaker: + - 0.1.0 Initial Chart + - 0.1.1 Change helm-toolkit dependency version to ">= 0.1.0" + - 0.1.2 Use full image ref for docker official images + - 0.1.3 Fix helm3 linting issue + - 0.1.4 Update htk requirements + - 0.1.5 Update default image value + - 0.1.6 Added OCI registry authentication + - 0.1.7 Use quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal by default + - 0.1.8 Update Chart.yaml apiVersion to v2 + - 2024.2.0 Update version to align with the Openstack release cycle +... diff --git a/roles/build-helm-packages/defaults/main.yml b/roles/build-helm-packages/defaults/main.yml new file mode 100644 index 0000000000..8e76d2ca61 --- /dev/null +++ b/roles/build-helm-packages/defaults/main.yml @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +version: + helm: v3.12.2 +url: + helm_repo: https://get.helm.sh +... diff --git a/roles/build-helm-packages/tasks/main.yaml b/roles/build-helm-packages/tasks/main.yaml new file mode 100644 index 0000000000..ef8cd1c450 --- /dev/null +++ b/roles/build-helm-packages/tasks/main.yaml @@ -0,0 +1,20 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- include: setup-helm-serve.yaml + +- name: build all charts in repo + make: + chdir: "{{ work_dir }}" + target: all +... diff --git a/roles/build-helm-packages/tasks/setup-helm-serve.yaml b/roles/build-helm-packages/tasks/setup-helm-serve.yaml new file mode 100644 index 0000000000..6e6ae7cc83 --- /dev/null +++ b/roles/build-helm-packages/tasks/setup-helm-serve.yaml @@ -0,0 +1,91 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- block: + - name: check if correct version of helm client already installed + shell: "set -e; [ \"x$($(type -p helm) version --client --short | awk '{ print $NF }' | awk -F '+' '{ print $1 }')\" == \"x${HELM_VERSION}\" ] || exit 1" + environment: + HELM_VERSION: "{{ version.helm }}" + args: + executable: /bin/bash + register: need_helm + ignore_errors: True + - name: install helm client + when: need_helm is failed + become_user: root + shell: | + TMP_DIR=$(mktemp -d) + curl -sSL ${HELM_REPO_URL}/helm-${HELM_VERSION}-linux-amd64.tar.gz | tar -zxv --strip-components=1 -C ${TMP_DIR} + sudo mv ${TMP_DIR}/helm /usr/bin/helm + rm -rf ${TMP_DIR} + environment: + HELM_VERSION: "{{ version.helm }}" + HELM_REPO_URL: "{{ url.helm_repo }}" + args: + executable: /bin/bash + - name: setting up helm client + command: helm init --client-only --skip-refresh --stable-repo-url "https://charts.helm.sh/stable" + +- block: + - name: checking if local helm server is running + shell: curl -s 127.0.0.1:8879 | grep -q 'Helm Repository' + args: + executable: /bin/bash + register: helm_server_running + ignore_errors: True + - name: getting current host user name + when: helm_server_running is failed + shell: id -un + args: + executable: /bin/bash + register: helm_server_user + - name: moving systemd unit into place for helm server + when: helm_server_running is failed + become: yes + become_user: root + template: + src: helm-serve.service.j2 + dest: /etc/systemd/system/helm-serve.service + mode: 416 + - name: starting helm serve service + when: helm_server_running is failed + become: yes + become_user: root + systemd: + state: restarted + daemon_reload: yes + name: helm-serve + enabled: yes + - name: wait for helm server to be ready + shell: curl -s 127.0.0.1:8879 | grep -q 'Helm Repository' + args: + executable: /bin/bash + register: wait_for_helm_server + until: wait_for_helm_server.rc == 0 + retries: 120 + delay: 5 + +- block: + - name: checking if helm 'stable' repo is present + shell: helm repo list | grep -q "^stable" + args: + executable: /bin/bash + register: helm_stable_repo_present + ignore_errors: True + - name: remove helm 'stable' repo when exists + when: helm_stable_repo_present is succeeded + command: helm repo remove stable + +- name: adding helm local repo + command: helm repo add local http://localhost:8879/charts +... diff --git a/roles/build-helm-packages/templates/helm-serve.service.j2 b/roles/build-helm-packages/templates/helm-serve.service.j2 new file mode 100644 index 0000000000..3cd1aad0f2 --- /dev/null +++ b/roles/build-helm-packages/templates/helm-serve.service.j2 @@ -0,0 +1,11 @@ +[Unit] +Description=Helm Server +After=network.target + +[Service] +User={{ helm_server_user.stdout }} +Restart=always +ExecStart=/usr/bin/helm serve + +[Install] +WantedBy=multi-user.target diff --git a/roles/clean-host/tasks/main.yaml b/roles/clean-host/tasks/main.yaml new file mode 100644 index 0000000000..9913ab14ac --- /dev/null +++ b/roles/clean-host/tasks/main.yaml @@ -0,0 +1,22 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: remove osh directory + become: yes + become_user: root + file: + path: "{{ item }}" + state: absent + with_items: + - /var/lib/openstack-helm +... diff --git a/roles/deploy-apparmor/tasks/main.yaml b/roles/deploy-apparmor/tasks/main.yaml new file mode 100644 index 0000000000..d00e7c8ad7 --- /dev/null +++ b/roles/deploy-apparmor/tasks/main.yaml @@ -0,0 +1,37 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- block: + - name: ensuring AppArmor is deployed on host + when: ansible_distribution == 'Ubuntu' + include_role: + name: deploy-package + tasks_from: dist + vars: + packages: + deb: + - apparmor + + - name: "Enable AppArmor" + when: ansible_distribution == 'Ubuntu' + become: true + become_user: root + shell: |- + set -xe + systemctl enable apparmor + systemctl start apparmor + systemctl status apparmor.service + args: + executable: /bin/bash + ignore_errors: True +... diff --git a/roles/deploy-docker/defaults/main.yml b/roles/deploy-docker/defaults/main.yml new file mode 100644 index 0000000000..b1a6fabd9c --- /dev/null +++ b/roles/deploy-docker/defaults/main.yml @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +proxy: + http: null + https: null + noproxy: null +... diff --git a/roles/deploy-docker/tasks/deploy-ansible-docker-support.yaml b/roles/deploy-docker/tasks/deploy-ansible-docker-support.yaml new file mode 100644 index 0000000000..ebbd244331 --- /dev/null +++ b/roles/deploy-docker/tasks/deploy-ansible-docker-support.yaml @@ -0,0 +1,69 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: ensuring SELinux is disabled on centos & fedora + when: ansible_distribution == 'CentOS' or ansible_distribution == 'Red Hat Enterprise Linux' or ansible_distribution == 'Fedora' + become: true + become_user: root + command: setenforce 0 + ignore_errors: True + +# NOTE(portdirect): See https://ask.openstack.org/en/question/110437/importerror-cannot-import-name-unrewindablebodyerror/ +- name: fix docker removal issue with ansible's docker_container on centos + when: ansible_distribution == 'CentOS' or ansible_distribution == 'Red Hat Enterprise Linux' + block: + - name: remove requests and urllib3 distro packages to fix docker removal issue with ansible's docker_container on centos + include_role: + name: deploy-package + tasks_from: dist + vars: + state: absent + packages: + rpm: + - python-urllib3 + - python-requests + - name: restore requests and urllib3 distro packages to fix docker removal issue with ansible's docker_container on centos + include_role: + name: deploy-package + tasks_from: dist + vars: + state: present + packages: + rpm: + - python-urllib3 + - python-requests + +- name: install additional packages + include_role: + name: deploy-package + tasks_from: dist + vars: + state: present + packages: + deb: + - conntrack + - bc + - nmap + rpm: + - conntrack-tools + - bc + - nmap + +- name: Ensure docker python packages deployed + include_role: + name: deploy-package + tasks_from: pip + vars: + packages: + - docker +... diff --git a/roles/deploy-docker/tasks/main.yaml b/roles/deploy-docker/tasks/main.yaml new file mode 100644 index 0000000000..d0ad154d2c --- /dev/null +++ b/roles/deploy-docker/tasks/main.yaml @@ -0,0 +1,80 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: setting default limit memlock + shell: | + set -xe; + echo "DefaultLimitMEMLOCK=16777216" | sudo tee -a /etc/systemd/system.conf + sudo systemctl daemon-reexec + sudo systemctl daemon-reload + +- name: check if docker deploy is needed + raw: which docker + register: need_docker + ignore_errors: True + +- name: centos | moving systemd unit into place + when: ( ansible_distribution == 'CentOS' or ansible_distribution == 'Red Hat Enterprise Linux' ) and ( need_docker is failed ) + template: + src: centos-docker.service.j2 + dest: /etc/systemd/system/docker.service + mode: 416 + +- name: fedora | moving systemd unit into place + when: ( ansible_distribution == 'Fedora' ) and ( need_docker is failed ) + template: + src: fedora-docker.service.j2 + dest: /etc/systemd/system/docker.service + mode: 416 + +- name: ubuntu | moving systemd unit into place + when: ( ansible_distribution == 'Debian' or ansible_distribution == 'Ubuntu' ) and ( need_docker is failed ) + template: + src: ubuntu-docker.service.j2 + dest: /etc/systemd/system/docker.service + mode: 416 + +# NOTE: (lamt) Setting up the proxy before installing docker +- name: ensure docker.service.d directory exists + when: proxy.http + file: + path: /etc/systemd/system/docker.service.d + state: directory + +- name: proxy | moving proxy systemd unit into place + when: proxy.http + template: + src: http-proxy.conf.j2 + dest: /etc/systemd/system/docker.service.d/http-proxy.conf + mode: 416 + +- name: deploy docker packages + when: need_docker is failed + include_role: + name: deploy-package + tasks_from: dist + vars: + packages: + deb: + - docker.io + rpm: + - docker + +- name: restarting docker + systemd: + state: restarted + daemon_reload: yes + name: docker + +- include: deploy-ansible-docker-support.yaml +... diff --git a/roles/deploy-docker/templates/centos-docker.service.j2 b/roles/deploy-docker/templates/centos-docker.service.j2 new file mode 100644 index 0000000000..bbaea27b85 --- /dev/null +++ b/roles/deploy-docker/templates/centos-docker.service.j2 @@ -0,0 +1,35 @@ +[Unit] +Description=Docker Application Container Engine +Documentation=http://docs.docker.com +After=network.target + +[Service] +Type=notify +NotifyAccess=all +Environment=GOTRACEBACK=crash +Environment=DOCKER_HTTP_HOST_COMPAT=1 +Environment=PATH=/usr/libexec/docker:/usr/bin:/usr/sbin +ExecStart=/usr/bin/dockerd-current \ + --add-runtime docker-runc=/usr/libexec/docker/docker-runc-current \ + --default-runtime=docker-runc \ + --exec-opt native.cgroupdriver=systemd \ + --userland-proxy-path=/usr/libexec/docker/docker-proxy-current \ + --seccomp-profile=/etc/docker/seccomp.json \ + --graph=/var/lib/docker \ + --storage-driver=overlay2 \ + --log-driver=json-file \ + --iptables=false +# NOTE(portdirect): fix mount propagation for CentOS, this is done post start, +# as docker seems to reset this. +ExecStartPost=/usr/bin/mount --make-rshared / +ExecReload=/bin/kill -s HUP $MAINPID +LimitNOFILE=1048576 +LimitNPROC=1048576 +LimitCORE=infinity +TimeoutStartSec=0 +Restart=on-abnormal +MountFlags=share +KillMode=process + +[Install] +WantedBy=multi-user.target diff --git a/roles/deploy-docker/templates/fedora-docker.service.j2 b/roles/deploy-docker/templates/fedora-docker.service.j2 new file mode 100644 index 0000000000..2c796c6be2 --- /dev/null +++ b/roles/deploy-docker/templates/fedora-docker.service.j2 @@ -0,0 +1,34 @@ +[Unit] +Description=Docker Application Container Engine +Documentation=http://docs.docker.com +After=network.target docker-containerd.service +Requires=docker-containerd.service + +[Service] +Type=notify +Environment=GOTRACEBACK=crash +ExecStart=/usr/bin/dockerd-current \ + --add-runtime oci=/usr/libexec/docker/docker-runc-current \ + --default-runtime=oci \ + --containerd /run/containerd.sock \ + --exec-opt native.cgroupdriver=systemd \ + --userland-proxy-path=/usr/libexec/docker/docker-proxy-current \ + --init-path=/usr/libexec/docker/docker-init-current \ + --seccomp-profile=/etc/docker/seccomp.json \ + --graph=/var/lib/docker \ + --storage-driver=overlay2 \ + --log-driver=json-file \ + --iptables=false +# NOTE(portdirect): fix mount propagation for Fedora, this is done post start, +# as docker seems to reset this. +ExecStartPost=/usr/bin/mount --make-rshared / +ExecReload=/bin/kill -s HUP $MAINPID +TasksMax=8192 +LimitNOFILE=1048576 +LimitNPROC=1048576 +LimitCORE=infinity +TimeoutStartSec=0 +Restart=on-abnormal + +[Install] +WantedBy=multi-user.target diff --git a/roles/deploy-docker/templates/http-proxy.conf.j2 b/roles/deploy-docker/templates/http-proxy.conf.j2 new file mode 100644 index 0000000000..90d8e1d534 --- /dev/null +++ b/roles/deploy-docker/templates/http-proxy.conf.j2 @@ -0,0 +1,4 @@ +[Service] +Environment="HTTP_PROXY={{ proxy.http }}" +Environment="HTTPS_PROXY={{ proxy.https }}" +Environment="NO_PROXY={{ proxy.noproxy }}" diff --git a/roles/deploy-docker/templates/ubuntu-docker.service.j2 b/roles/deploy-docker/templates/ubuntu-docker.service.j2 new file mode 100644 index 0000000000..2451b19803 --- /dev/null +++ b/roles/deploy-docker/templates/ubuntu-docker.service.j2 @@ -0,0 +1,30 @@ +[Unit] +Description=Docker Application Container Engine +Documentation=https://docs.docker.com +After=network.target docker.socket firewalld.service +Requires=docker.socket + +[Service] +Type=notify +# the default is not to use systemd for cgroups because the delegate issues still +# exists and systemd currently does not support the cgroup feature set required +# for containers run by docker +EnvironmentFile=-/etc/default/docker +ExecStart=/usr/bin/dockerd --iptables=false -H fd:// $DOCKER_OPTS +ExecReload=/bin/kill -s HUP $MAINPID +LimitNOFILE=1048576 +# Having non-zero Limit*s causes performance problems due to accounting overhead +# in the kernel. We recommend using cgroups to do container-local accounting. +LimitNPROC=infinity +LimitCORE=infinity +# Uncomment TasksMax if your systemd version supports it. +# Only systemd 226 and above support this version. +TasksMax=infinity +TimeoutStartSec=0 +# set delegate yes so that systemd does not reset the cgroups of docker containers +Delegate=yes +# kill only the docker process, not all processes in the cgroup +KillMode=process + +[Install] +WantedBy=multi-user.target diff --git a/roles/deploy-env/README.md b/roles/deploy-env/README.md new file mode 100644 index 0000000000..116fff98e0 --- /dev/null +++ b/roles/deploy-env/README.md @@ -0,0 +1,59 @@ +This role is used to deploy test environment which includes +- install necessary prerequisites including Helm +- deploy Containerd and a container runtime for Kubernetes +- deploy Kubernetes using Kubeadm with a single control plane node +- install Calico as a Kubernetes networking +- establish tunnel between primary node and K8s control plane ndoe + +The role works both for single-node and multi-node inventories. The role +totally relies on inventory groups. The `primary` and `k8s_control_plane` +groups must include only one node and this can be the same node for these two +groups. + +The `primary` group is where we install `kubectl` and `helm` CLI tools. +You can consider this group as a deployer's machine. + +The `k8s_control_plane` is where we deploy the K8s control plane. + +The `k8s_cluster` group must include all the K8s nodes including control plane +and worker nodes. + +In case of running tests on a single-node environment the group `k8s_nodes` +must be empty. This means the K8s cluster will consist of a single control plane +node where all the workloads will be running. + +See for example: + +```yaml +all: + vars: + ansible_port: 22 + ansible_user: ubuntu + ansible_ssh_private_key_file: /home/ubuntu/.ssh/id_rsa + ansible_ssh_extra_args: -o StrictHostKeyChecking=no + hosts: + primary: + ansible_host: 10.10.10.10 + node-1: + ansible_host: 10.10.10.11 + node-2: + ansible_host: 10.10.10.12 + node-3: + ansible_host: 10.10.10.13 + children: + primary: + hosts: + primary: + k8s_cluster: + hosts: + node-1: + node-2: + node-3: + k8s_control_plane: + hosts: + node-1: + k8s_nodes: + hosts: + node-2: + node-3: +``` diff --git a/roles/deploy-env/defaults/main.yaml b/roles/deploy-env/defaults/main.yaml new file mode 100644 index 0000000000..f1107b6fe8 --- /dev/null +++ b/roles/deploy-env/defaults/main.yaml @@ -0,0 +1,77 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +--- +kube_version_repo: "v1.31" +# the list of k8s package versions are available here +# https://pkgs.k8s.io/core:/stable:/{{ kube_version_repo }}/deb/Packages +kube_version: "1.31.3-1.1" +helm_version: "v3.14.0" +crictl_version: "v1.30.1" + +calico_setup: true +calico_version: "v3.27.4" +calico_manifest_url: "https://raw.githubusercontent.com/projectcalico/calico/{{ calico_version }}/manifests/calico.yaml" + +cilium_setup: false +cilium_version: "1.16.0" + +flannel_setup: false +flannel_version: v0.25.4 + +ingress_setup: false +ingress_nginx_version: "4.8.3" +ingress_openstack_setup: true +ingress_ceph_setup: true +ingress_osh_infra_setup: false + +kubectl: + user: zuul + group: zuul + +osh_plugin_repo: "https://opendev.org/openstack/openstack-helm-plugin.git" + +kubeadm: + pod_network_cidr: "10.244.0.0/16" + service_cidr: "10.96.0.0/16" +docker: + root_path: /var/lib/docker +docker_users: + - zuul +containerd: + root_path: /var/lib/containerd +loopback_setup: false +loopback_device: /dev/loop100 +loopback_image: /var/lib/openstack-helm/ceph-loop.img +loopback_image_size: 12G + +coredns_resolver_setup: true + +metallb_setup: false +metallb_version: "0.13.12" +metallb_pool_cidr: "172.24.128.0/24" +metallb_openstack_endpoint_cidr: "172.24.128.100/24" + +client_cluster_ssh_setup: true +client_ssh_user: zuul +cluster_ssh_user: zuul + +openstack_provider_gateway_setup: false +openstack_provider_network_cidr: "172.24.4.0/24" +openstack_provider_gateway_cidr: "172.24.4.1/24" + +tunnel_network_cidr: "172.24.5.0/24" +tunnel_client_cidr: "172.24.5.2/24" +tunnel_cluster_cidr: "172.24.5.1/24" + +dnsmasq_image: "quay.io/airshipit/neutron:2024.2-ubuntu_jammy" +nginx_image: "quay.io/airshipit/nginx:alpine3.18" +... diff --git a/roles/deploy-env/files/calico_patch.yaml b/roles/deploy-env/files/calico_patch.yaml new file mode 100644 index 0000000000..bdada7422d --- /dev/null +++ b/roles/deploy-env/files/calico_patch.yaml @@ -0,0 +1,22 @@ +--- +spec: + template: + metadata: + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9091" + spec: + containers: + - name: calico-node + env: + - name: FELIX_PROMETHEUSMETRICSENABLED + value: "true" + - name: FELIX_PROMETHEUSMETRICSPORT + value: "9091" + - name: FELIX_IGNORELOOSERPF + value: "true" + # we need Calico to skip this interface while discovering the + # network changes on the host to prevent announcing unnecessary networks. + - name: IP_AUTODETECTION_METHOD + value: "skip-interface=br-ex|provider.*|client.*" +... diff --git a/roles/deploy-env/files/cluster_resolv.conf b/roles/deploy-env/files/cluster_resolv.conf new file mode 100644 index 0000000000..eb91d36e03 --- /dev/null +++ b/roles/deploy-env/files/cluster_resolv.conf @@ -0,0 +1 @@ +nameserver 10.96.0.10 diff --git a/roles/deploy-env/files/containerd_config.toml b/roles/deploy-env/files/containerd_config.toml new file mode 100644 index 0000000000..b2868dc0f0 --- /dev/null +++ b/roles/deploy-env/files/containerd_config.toml @@ -0,0 +1,256 @@ +disabled_plugins = [] +imports = [] +oom_score = 0 +plugin_dir = "" +required_plugins = [] +root = "{{ containerd.root_path }}" +state = "/run/containerd" +temp = "" +version = 2 + +[cgroup] + path = "" + +[debug] + address = "" + format = "" + gid = 0 + level = "" + uid = 0 + +[grpc] + address = "/run/containerd/containerd.sock" + gid = 0 + max_recv_message_size = 16777216 + max_send_message_size = 16777216 + tcp_address = "" + tcp_tls_ca = "" + tcp_tls_cert = "" + tcp_tls_key = "" + uid = 0 + +[metrics] + address = "" + grpc_histogram = false + +[plugins] + + [plugins."io.containerd.gc.v1.scheduler"] + deletion_threshold = 0 + mutation_threshold = 100 + pause_threshold = 0.02 + schedule_delay = "0s" + startup_delay = "100ms" + + [plugins."io.containerd.grpc.v1.cri"] + device_ownership_from_security_context = false + disable_apparmor = false + disable_cgroup = false + disable_hugetlb_controller = true + disable_proc_mount = false + disable_tcp_service = true + enable_selinux = false + enable_tls_streaming = false + enable_unprivileged_icmp = false + enable_unprivileged_ports = false + ignore_image_defined_volumes = false + max_concurrent_downloads = 3 + max_container_log_line_size = 16384 + netns_mounts_under_state_dir = false + restrict_oom_score_adj = false + sandbox_image = "registry.k8s.io/pause:3.9" + selinux_category_range = 1024 + stats_collect_period = 10 + stream_idle_timeout = "4h0m0s" + stream_server_address = "127.0.0.1" + stream_server_port = "0" + systemd_cgroup = false + tolerate_missing_hugetlb_controller = true + unset_seccomp_profile = "" + + [plugins."io.containerd.grpc.v1.cri".cni] + bin_dir = "/opt/cni/bin" + conf_dir = "/etc/cni/net.d" + conf_template = "" + ip_pref = "" + max_conf_num = 1 + + [plugins."io.containerd.grpc.v1.cri".containerd] + default_runtime_name = "runc" + disable_snapshot_annotations = true + discard_unpacked_layers = false + ignore_rdt_not_enabled_errors = false + no_pivot = false + snapshotter = "overlayfs" + + [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime] + base_runtime_spec = "" + cni_conf_dir = "" + cni_max_conf_num = 0 + container_annotations = [] + pod_annotations = [] + privileged_without_host_devices = false + runtime_engine = "" + runtime_path = "" + runtime_root = "" + runtime_type = "" + + [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime.options] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes] + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc] + base_runtime_spec = "" + cni_conf_dir = "" + cni_max_conf_num = 0 + container_annotations = [] + pod_annotations = [] + privileged_without_host_devices = false + runtime_engine = "" + runtime_path = "" + runtime_root = "" + runtime_type = "io.containerd.runc.v2" + + [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options] + BinaryName = "" + CriuImagePath = "" + CriuPath = "" + CriuWorkPath = "" + IoGid = 0 + IoUid = 0 + NoNewKeyring = false + NoPivotRoot = false + Root = "" + ShimCgroup = "" + SystemdCgroup = true + + [plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime] + base_runtime_spec = "" + cni_conf_dir = "" + cni_max_conf_num = 0 + container_annotations = [] + pod_annotations = [] + privileged_without_host_devices = false + runtime_engine = "" + runtime_path = "" + runtime_root = "" + runtime_type = "" + + [plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime.options] + + [plugins."io.containerd.grpc.v1.cri".image_decryption] + key_model = "node" + + [plugins."io.containerd.grpc.v1.cri".registry] + config_path = "/etc/containerd/certs.d" + + [plugins."io.containerd.grpc.v1.cri".registry.auths] + + [plugins."io.containerd.grpc.v1.cri".registry.configs] +{% for item in registry_namespaces %} +{% if item.auth is defined %} + [plugins."io.containerd.grpc.v1.cri".registry.configs."{{ item.namespace }}".auth] + auth = "{{ item.auth }}" +{% endif %} +{% endfor %} + + [plugins."io.containerd.grpc.v1.cri".registry.headers] + + [plugins."io.containerd.grpc.v1.cri".registry.mirrors] + + [plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming] + tls_cert_file = "" + tls_key_file = "" + + [plugins."io.containerd.internal.v1.opt"] + path = "/opt/containerd" + + [plugins."io.containerd.internal.v1.restart"] + interval = "10s" + + [plugins."io.containerd.internal.v1.tracing"] + sampling_ratio = 1.0 + service_name = "containerd" + + [plugins."io.containerd.metadata.v1.bolt"] + content_sharing_policy = "shared" + + [plugins."io.containerd.monitor.v1.cgroups"] + no_prometheus = false + + [plugins."io.containerd.runtime.v1.linux"] + no_shim = false + runtime = "runc" + runtime_root = "" + shim = "containerd-shim" + shim_debug = false + + [plugins."io.containerd.runtime.v2.task"] + platforms = ["linux/amd64"] + sched_core = false + + [plugins."io.containerd.service.v1.diff-service"] + default = ["walking"] + + [plugins."io.containerd.service.v1.tasks-service"] + rdt_config_file = "" + + [plugins."io.containerd.snapshotter.v1.aufs"] + root_path = "" + + [plugins."io.containerd.snapshotter.v1.btrfs"] + root_path = "" + + [plugins."io.containerd.snapshotter.v1.devmapper"] + async_remove = false + base_image_size = "" + discard_blocks = false + fs_options = "" + fs_type = "" + pool_name = "" + root_path = "" + + [plugins."io.containerd.snapshotter.v1.native"] + root_path = "" + + [plugins."io.containerd.snapshotter.v1.overlayfs"] + root_path = "" + upperdir_label = false + + [plugins."io.containerd.snapshotter.v1.zfs"] + root_path = "" + + [plugins."io.containerd.tracing.processor.v1.otlp"] + endpoint = "" + insecure = false + protocol = "" + +[proxy_plugins] + +[stream_processors] + + [stream_processors."io.containerd.ocicrypt.decoder.v1.tar"] + accepts = ["application/vnd.oci.image.layer.v1.tar+encrypted"] + args = ["--decryption-keys-path", "/etc/containerd/ocicrypt/keys"] + env = ["OCICRYPT_KEYPROVIDER_CONFIG=/etc/containerd/ocicrypt/ocicrypt_keyprovider.conf"] + path = "ctd-decoder" + returns = "application/vnd.oci.image.layer.v1.tar" + + [stream_processors."io.containerd.ocicrypt.decoder.v1.tar.gzip"] + accepts = ["application/vnd.oci.image.layer.v1.tar+gzip+encrypted"] + args = ["--decryption-keys-path", "/etc/containerd/ocicrypt/keys"] + env = ["OCICRYPT_KEYPROVIDER_CONFIG=/etc/containerd/ocicrypt/ocicrypt_keyprovider.conf"] + path = "ctd-decoder" + returns = "application/vnd.oci.image.layer.v1.tar+gzip" + +[timeouts] + "io.containerd.timeout.bolt.open" = "0s" + "io.containerd.timeout.shim.cleanup" = "5s" + "io.containerd.timeout.shim.load" = "5s" + "io.containerd.timeout.shim.shutdown" = "3s" + "io.containerd.timeout.task.state" = "2s" + +[ttrpc] + address = "" + gid = 0 + uid = 0 diff --git a/roles/deploy-env/files/daemon.json b/roles/deploy-env/files/daemon.json new file mode 100644 index 0000000000..29325b352e --- /dev/null +++ b/roles/deploy-env/files/daemon.json @@ -0,0 +1,16 @@ +{ + "data-root": "{{ docker.root_path }}", + "exec-opts": ["native.cgroupdriver=systemd"], + "log-driver": "json-file", + "log-opts": { + "max-size": "100m" + }, +{% if registry_mirror is defined %} + "registry-mirrors": ["{{ registry_mirror }}"], +{% endif %} +{% if insecure_registries is defined %} + "insecure-registries": ["{{ insecure_registries }}"], +{% endif %} + "storage-driver": "overlay2", + "live-restore": true +} diff --git a/roles/deploy-env/files/hosts b/roles/deploy-env/files/hosts new file mode 100644 index 0000000000..dea9afeb93 --- /dev/null +++ b/roles/deploy-env/files/hosts @@ -0,0 +1,5 @@ +127.0.0.1 localhost +{{ ansible_default_ipv4['address'] }} {{ ansible_hostname }} +{% if buildset_registry is defined and (buildset_registry.host | ipaddr) %} +{{ buildset_registry.host }} zuul-jobs.buildset-registry +{% endif %} diff --git a/roles/deploy-env/files/hosts.toml b/roles/deploy-env/files/hosts.toml new file mode 100644 index 0000000000..e8c08eedbb --- /dev/null +++ b/roles/deploy-env/files/hosts.toml @@ -0,0 +1,12 @@ +{% if item.skip_server is not defined or not item.skip_server %} +server = "{{ item.server | default('https://' + item.namespace) }}" +{% endif %} + +[host."{{ item.mirror }}"] +capabilities = ["pull", "resolve", "push"] +{% if item.ca is defined %} +ca = "{{ item.ca }}" +{% endif %} +{% if item.skip_verify is defined and item.skip_verify %} +skip_verify = true +{% endif %} diff --git a/roles/deploy-env/files/kubeadm_config.yaml b/roles/deploy-env/files/kubeadm_config.yaml new file mode 100644 index 0000000000..137e0781a5 --- /dev/null +++ b/roles/deploy-env/files/kubeadm_config.yaml @@ -0,0 +1,27 @@ +--- +apiVersion: kubeproxy.config.k8s.io/v1alpha1 +kind: KubeProxyConfiguration +mode: ipvs +ipvs: + strictARP: true +... +--- +apiVersion: kubeadm.k8s.io/v1beta3 +kind: ClusterConfiguration +networking: + serviceSubnet: "{{ kubeadm.service_cidr }}" # --service-cidr + podSubnet: "{{ kubeadm.pod_network_cidr }}" # --pod-network-cidr + dnsDomain: "cluster.local" +... +--- +apiVersion: kubeadm.k8s.io/v1beta3 +kind: InitConfiguration +nodeRegistration: + taints: [] +... +--- +apiVersion: kubeadm.k8s.io/v1beta3 +kind: JoinConfiguration +nodeRegistration: + taints: [] +... diff --git a/roles/deploy-env/files/loop-setup.service b/roles/deploy-env/files/loop-setup.service new file mode 100644 index 0000000000..d4d6e3f09e --- /dev/null +++ b/roles/deploy-env/files/loop-setup.service @@ -0,0 +1,18 @@ +[Unit] +Description=Setup loop devices +DefaultDependencies=no +Conflicts=umount.target +Before=local-fs.target +After=systemd-udevd.service +Requires=systemd-udevd.service + +[Service] +Type=oneshot +ExecStart=/sbin/losetup {{ loopback_device }} '{{ loopback_image }}' +ExecStop=/sbin/losetup -d {{ loopback_device }} +TimeoutSec=60 +RemainAfterExit=yes + +[Install] +WantedBy=local-fs.target +Also=systemd-udevd.service diff --git a/roles/deploy-env/files/nginx_tcp_proxy.conf b/roles/deploy-env/files/nginx_tcp_proxy.conf new file mode 100644 index 0000000000..3d64369dff --- /dev/null +++ b/roles/deploy-env/files/nginx_tcp_proxy.conf @@ -0,0 +1,25 @@ +user nginx; +worker_processes auto; + +error_log /dev/stdout warn; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +stream { + access_log off; + + server { + listen {{ openstack_provider_gateway_cidr | ipaddr('address') }}:80; + proxy_pass {{ metallb_openstack_endpoint_cidr | ipaddr('address') }}:80; + proxy_bind {{ openstack_provider_gateway_cidr | ipaddr('address') }} transparent; + } + + server { + listen {{ openstack_provider_gateway_cidr | ipaddr('address') }}:443; + proxy_pass {{ metallb_openstack_endpoint_cidr | ipaddr('address') }}:443; + proxy_bind {{ openstack_provider_gateway_cidr | ipaddr('address') }} transparent; + } +} diff --git a/roles/deploy-env/files/resolv.conf b/roles/deploy-env/files/resolv.conf new file mode 100644 index 0000000000..12f0168938 --- /dev/null +++ b/roles/deploy-env/files/resolv.conf @@ -0,0 +1 @@ +nameserver {{ nameserver_ip }} diff --git a/roles/deploy-env/files/ssh_config b/roles/deploy-env/files/ssh_config new file mode 100644 index 0000000000..a9ecad07c3 --- /dev/null +++ b/roles/deploy-env/files/ssh_config @@ -0,0 +1 @@ +StrictHostKeyChecking no diff --git a/roles/deploy-env/handlers/main.yaml b/roles/deploy-env/handlers/main.yaml new file mode 100644 index 0000000000..60d2ef542c --- /dev/null +++ b/roles/deploy-env/handlers/main.yaml @@ -0,0 +1,21 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Systemd reload + shell: systemctl daemon-reload + +- name: Restart loop-setup + service: + name: loop-setup + state: restarted +... diff --git a/roles/deploy-env/tasks/buildset_registry_alias.yaml b/roles/deploy-env/tasks/buildset_registry_alias.yaml new file mode 100644 index 0000000000..163eb84f4b --- /dev/null +++ b/roles/deploy-env/tasks/buildset_registry_alias.yaml @@ -0,0 +1,25 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Set buildset_registry alias variable when using ip + set_fact: + buildset_registry_alias: zuul-jobs.buildset-registry + when: + - buildset_registry.host | ipaddr + +- name: Set buildset_registry alias variable when using name + set_fact: + buildset_registry_alias: "{{ buildset_registry.host }}" + when: + - not ( buildset_registry.host | ipaddr ) +... diff --git a/roles/deploy-env/tasks/calico.yaml b/roles/deploy-env/tasks/calico.yaml new file mode 100644 index 0000000000..f79d6311c4 --- /dev/null +++ b/roles/deploy-env/tasks/calico.yaml @@ -0,0 +1,55 @@ +--- +# We download Calico manifest on all nodes because we then want to download +# Calico images BEFORE deploying it, so that `kubectl wait` timeout +# for `k8s-app=kube-dns` isn't reached by slow download speeds +- name: Download Calico manifest + when: inventory_hostname in (groups['k8s_cluster'] | default([])) + shell: | + curl -LSs {{ calico_manifest_url }} -o /tmp/calico.yaml + sed -i -e 's#docker.io/calico/#quay.io/calico/#g' /tmp/calico.yaml + export CONTAINER_RUNTIME_ENDPOINT=unix:///run/containerd/containerd.sock + export IMAGE_SERVICE_ENDPOINT=unix:///run/containerd/containerd.sock + awk '/image:/ { print $2 }' /tmp/calico.yaml | xargs -I{} crictl pull {} + args: + executable: /bin/bash + +- name: Deploy Calico + become: false + when: inventory_hostname in (groups['primary'] | default([])) + block: + - name: Download Calico manifest + shell: | + if [[ ! -f /tmp/calico.yaml ]]; then + curl -LSs {{ calico_manifest_url }} -o /tmp/calico.yaml + sed -i -e 's#docker.io/calico/#quay.io/calico/#g' /tmp/calico.yaml + fi + args: + executable: /bin/bash + + - name: Deploy Calico + command: kubectl apply -f /tmp/calico.yaml + + - name: Sleep before trying to check Calico pods + pause: + seconds: 30 + + - name: Wait for Calico pods ready + command: kubectl -n kube-system wait --timeout=20s --for=condition=Ready pods -l k8s-app=calico-node + register: calico_pods_wait + until: calico_pods_wait is succeeded + retries: 10 + + - name: Prepare Calico patch + copy: + src: files/calico_patch.yaml + dest: /tmp/calico_patch.yaml + + - name: Patch Calico + command: kubectl -n kube-system patch daemonset calico-node --patch-file /tmp/calico_patch.yaml + + - name: Wait for Calico pods ready (after patch) + command: kubectl -n kube-system wait --timeout=20s --for=condition=Ready pods -l k8s-app=calico-node + register: calico_pods_wait + until: calico_pods_wait is succeeded + retries: 10 +... diff --git a/roles/deploy-env/tasks/cilium.yaml b/roles/deploy-env/tasks/cilium.yaml new file mode 100644 index 0000000000..b27d85eb0c --- /dev/null +++ b/roles/deploy-env/tasks/cilium.yaml @@ -0,0 +1,22 @@ +--- +- name: Download Cilium + shell: | + CILIUM_CLI_VERSION=$(curl -s https://raw.githubusercontent.com/cilium/cilium-cli/main/stable.txt) + CLI_ARCH=amd64 + curl -L --fail --remote-name-all https://github.com/cilium/cilium-cli/releases/download/${CILIUM_CLI_VERSION}/cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum} + sha256sum --check cilium-linux-${CLI_ARCH}.tar.gz.sha256sum + tar xzvfC cilium-linux-${CLI_ARCH}.tar.gz /usr/local/bin + rm cilium-linux-${CLI_ARCH}.tar.gz{,.sha256sum} + args: + executable: /bin/bash + chdir: /tmp + when: inventory_hostname in (groups['primary'] | default([])) + +- name: Deploy Cilium + become: false + shell: | + cilium install --version {{ cilium_version }} + args: + executable: /bin/bash + when: inventory_hostname in (groups['primary'] | default([])) +... diff --git a/roles/deploy-env/tasks/client_cluster_ssh.yaml b/roles/deploy-env/tasks/client_cluster_ssh.yaml new file mode 100644 index 0000000000..c8a07a1bb2 --- /dev/null +++ b/roles/deploy-env/tasks/client_cluster_ssh.yaml @@ -0,0 +1,72 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Set client user home directory + set_fact: + client_user_home_directory: /home/{{ client_ssh_user }} + when: client_ssh_user != "root" + +- name: Set client user home directory + set_fact: + client_user_home_directory: /root + when: client_ssh_user == "root" + +- name: Set cluster user home directory + set_fact: + cluster_user_home_directory: /home/{{ cluster_ssh_user }} + when: cluster_ssh_user != "root" + +- name: Set cluster user home directory + set_fact: + cluster_user_home_directory: /root + when: cluster_ssh_user == "root" + +- name: Setup ssh keys + become_user: "{{ client_ssh_user }}" + block: + - name: Generate ssh key pair + shell: | + ssh-keygen -t ed25519 -q -N "" -f {{ client_user_home_directory }}/.ssh/id_ed25519 + args: + creates: "{{ client_user_home_directory }}/.ssh/id_ed25519.pub" + when: (inventory_hostname in (groups['primary'] | default([]))) + + - name: Read ssh public key + command: cat "{{ client_user_home_directory }}/.ssh/id_ed25519.pub" + register: ssh_public_key + when: (inventory_hostname in (groups['primary'] | default([]))) + +- name: Setup passwordless ssh from primary and cluster nodes + become_user: "{{ cluster_ssh_user }}" + block: + - name: Set primary ssh public key + set_fact: + client_ssh_public_key: "{{ (groups['primary'] | map('extract', hostvars, ['ssh_public_key', 'stdout']))[0] }}" + when: inventory_hostname in (groups['k8s_cluster'] | default([])) + + - name: Put keys to .ssh/authorized_keys + lineinfile: + path: "{{ cluster_user_home_directory }}/.ssh/authorized_keys" + state: present + line: "{{ client_ssh_public_key }}" + when: inventory_hostname in (groups['k8s_cluster'] | default([])) + + - name: Disable strict host key checking + template: + src: "files/ssh_config" + dest: "{{ client_user_home_directory }}/.ssh/config" + owner: "{{ client_ssh_user }}" + mode: 0644 + backup: true + when: (inventory_hostname in (groups['primary'] | default([]))) +... diff --git a/roles/deploy-env/tasks/client_cluster_tunnel.yaml b/roles/deploy-env/tasks/client_cluster_tunnel.yaml new file mode 100644 index 0000000000..31d3118b3a --- /dev/null +++ b/roles/deploy-env/tasks/client_cluster_tunnel.yaml @@ -0,0 +1,76 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Set cluster IP + set_fact: + cluster_default_ip: "{{ (groups['k8s_control_plane'] | map('extract', hostvars, ['ansible_default_ipv4', 'address']))[0] }}" + +- name: Set client IP + set_fact: + client_default_ip: "{{ (groups['primary'] | map('extract', hostvars, ['ansible_default_ipv4', 'address']))[0] }}" + +- name: Setup wireguard keys + when: (groups['primary'] | difference(groups['k8s_control_plane']) | length > 0) + block: + - name: Generate wireguard key pair + shell: | + wg genkey | tee /root/wg-private-key | wg pubkey > /root/wg-public-key + chmod 600 /root/wg-private-key + when: (inventory_hostname in (groups['primary'] | default([]))) or (inventory_hostname in (groups['k8s_control_plane'] | default([]))) + + - name: Register public wireguard key variable + command: cat /root/wg-public-key + register: wg_public_key + when: (inventory_hostname in (groups['primary'] | default([]))) or (inventory_hostname in (groups['k8s_control_plane'] | default([]))) + +- name: Setup wireguard tunnel between primary and cluster control-plane node + when: (groups['primary'] | difference(groups['k8s_control_plane']) | length > 0) + block: + - name: Set primary wireguard public key + set_fact: + client_wg_public_key: "{{ (groups['primary'] | map('extract', hostvars, ['wg_public_key', 'stdout']))[0] }}" + when: inventory_hostname in (groups['k8s_control_plane'] | default([])) + + - name: Set cluster wireguard public key + set_fact: + cluster_wg_public_key: "{{ (groups['k8s_control_plane'] | map('extract', hostvars, ['wg_public_key', 'stdout']))[0] }}" + when: inventory_hostname in (groups['primary'] | default([])) + + - name: Set up wireguard tunnel on cluster control-plane node + shell: | + cat > /tmp/configure_cluster_tunnel.sh < /tmp/configure_client_tunnel.sh < /tmp/coredns_configmap.yaml < 0 + +- name: Install Kubernetes binaries + apt: + state: present + update_cache: true + allow_downgrade: true + pkg: + - "kubelet={{ kube_version }}" + - "kubeadm={{ kube_version }}" + - "kubectl={{ kube_version }}" + +- name: Restart kubelet + service: + name: kubelet + daemon_reload: yes + state: restarted + +- name: Configure resolv.conf + template: + src: files/resolv.conf + dest: /etc/resolv.conf + owner: root + group: root + mode: 0644 + vars: + nameserver_ip: "8.8.8.8" + +- name: Disable systemd-resolved + service: + name: systemd-resolved + enabled: false + state: stopped + ignore_errors: true + +- name: Disable unbound + service: + name: unbound + enabled: false + state: stopped + ignore_errors: true +... diff --git a/roles/deploy-env/tasks/k8s_control_plane.yaml b/roles/deploy-env/tasks/k8s_control_plane.yaml new file mode 100644 index 0000000000..563f09b2d6 --- /dev/null +++ b/roles/deploy-env/tasks/k8s_control_plane.yaml @@ -0,0 +1,39 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Mount tmpfs to /var/lib/etcd + mount: + path: /var/lib/etcd + src: tmpfs + fstype: tmpfs + opts: size=1g + state: mounted + +- name: Prepare kubeadm config + template: + src: files/kubeadm_config.yaml + dest: /tmp/kubeadm_config.yaml + +- name: Initialize the Kubernetes cluster using kubeadm + command: kubeadm init --config /tmp/kubeadm_config.yaml + +- name: Generate join command + command: kubeadm token create --print-join-command + register: join_command + +- name: "Copy kube config to localhost" + synchronize: + mode: pull + src: /etc/kubernetes/admin.conf + dest: /tmp/kube_config +... diff --git a/roles/deploy-env/tasks/loopback_devices.yaml b/roles/deploy-env/tasks/loopback_devices.yaml new file mode 100644 index 0000000000..c15288cdf8 --- /dev/null +++ b/roles/deploy-env/tasks/loopback_devices.yaml @@ -0,0 +1,45 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Create loop device image + shell: | + mkdir -p {{ loopback_image | dirname }} + truncate -s {{ loopback_image_size }} {{ loopback_image }} + +- name: Create loop device + shell: | + mknod {{ loopback_device }} b $(grep loop /proc/devices | cut -c3) {{ loopback_device | regex_search('[0-9]+') }} + +- name: Create loop-setup systemd unit + template: + src: files/loop-setup.service + dest: /etc/systemd/system/loop-setup.service + notify: + - Systemd reload + +- name: Systemd reload + shell: systemctl daemon-reload + +- name: Configure loop-setup systemd unit + service: + name: loop-setup + enabled: yes + state: started + notify: + - Systemd reload + - Restart loop-setup + +- name: Check {{ loopback_device }} is attached + shell: | + losetup | grep -i {{ loopback_device }} +... diff --git a/roles/deploy-env/tasks/main.yaml b/roles/deploy-env/tasks/main.yaml new file mode 100644 index 0000000000..d1caef39ae --- /dev/null +++ b/roles/deploy-env/tasks/main.yaml @@ -0,0 +1,106 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Include prerequisites tasks + include_tasks: + file: prerequisites.yaml + +- name: Configure /etc/hosts + template: + src: files/hosts + dest: /etc/hosts + +- name: Loop devices + include_tasks: + file: loopback_devices.yaml + when: loopback_setup and inventory_hostname in (groups['k8s_cluster'] | default([])) + +- name: Deploy Containerd + include_tasks: + file: containerd.yaml + +- name: Include K8s common tasks + include_tasks: + file: k8s_common.yaml + when: inventory_hostname in (groups['k8s_cluster'] | default([])) + +- name: Include K8s control-plane tasks + include_tasks: + file: k8s_control_plane.yaml + when: inventory_hostname in (groups['k8s_control_plane'] | default([])) + +- name: Join workload nodes to cluster + command: "{{ (groups['k8s_control_plane'] | map('extract', hostvars, ['join_command', 'stdout_lines', 0]))[0] }}" + when: inventory_hostname in (groups['k8s_nodes'] | default([])) + +- name: Include K8s client tasks + include_tasks: + file: k8s_client.yaml + when: inventory_hostname in (groups['primary'] | default([])) + +- name: Include Calico tasks + include_tasks: + file: calico.yaml + when: calico_setup + +- name: Include Cilium tasks + include_tasks: + file: cilium.yaml + when: cilium_setup + +- name: Include Flannel tasks + include_tasks: + file: flannel.yaml + when: flannel_setup + +- name: Include coredns resolver tasks + include_tasks: + file: coredns_resolver.yaml + when: coredns_resolver_setup + +- name: Include Openstack provider gateway tasks + include_tasks: + file: openstack_provider_gateway.yaml + when: + - openstack_provider_gateway_setup + - inventory_hostname in (groups['k8s_control_plane'] | default([])) + +- name: Include Metallb tasks + include_tasks: + file: metallb.yaml + when: metallb_setup + +- name: Include Openstack Metallb endpoint tasks + include_tasks: + file: openstack_metallb_endpoint.yaml + when: + - metallb_setup + - inventory_hostname in (groups['primary'] | default([])) + +- name: Include client-to-cluster tunnel tasks + include_tasks: + file: client_cluster_tunnel.yaml + when: (groups['primary'] | difference(groups['k8s_control_plane']) | length > 0) + +- name: Include client-to-cluster ssh key tasks + include_tasks: + file: client_cluster_ssh.yaml + when: client_cluster_ssh_setup + +- name: Include ingress tasks + include_tasks: + file: ingress.yaml + when: + - ingress_setup + - inventory_hostname in (groups['primary'] | default([])) +... diff --git a/roles/deploy-env/tasks/metallb.yaml b/roles/deploy-env/tasks/metallb.yaml new file mode 100644 index 0000000000..01c3264348 --- /dev/null +++ b/roles/deploy-env/tasks/metallb.yaml @@ -0,0 +1,65 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Deploy MetalLB + become: false + when: inventory_hostname in (groups['primary'] | default([])) + block: + - name: Add MetalLB chart repo + become_user: "{{ kubectl.user }}" + shell: | + helm repo add metallb https://metallb.github.io/metallb + + - name: Install MetalLB + become_user: "{{ kubectl.user }}" + shell: | + helm upgrade --install metallb metallb/metallb \ + --version {{ metallb_version }} \ + --namespace metallb-system \ + --create-namespace + + - name: Sleep before trying to check MetalLB pods + pause: + seconds: 30 + + - name: Wait for MetalLB pods ready + command: kubectl -n metallb-system wait --timeout=240s --for=condition=Ready pods -l 'app.kubernetes.io/name=metallb' + + - name: Create MetalLB address pool + shell: | + tee > /tmp/metallb_ipaddresspool.yaml < /tmp/metallb_l2advertisement.yaml < /tmp/openstack_endpoint_service.yaml < "${DIR}/${NAME}.yaml" + kubectl describe ${OBJECT} ${NAME} > "${DIR}/${NAME}.txt" + } + export -f get_objects + + list_objects | \ + xargs -r -n 1 -P ${PARALLELISM_FACTOR} -I {} bash -c 'name_objects "$@"' _ {} | \ + xargs -r -n 1 -P ${PARALLELISM_FACTOR} -I {} bash -c 'get_objects "$@"' _ {} + args: + executable: /bin/bash + ignore_errors: True + +- name: "creating directory for namespace scoped objects" + file: + path: "{{ logs_dir }}/objects/namespaced" + state: directory + +- name: "Gathering descriptions for namespace scoped objects" + shell: |- + set -e + export OBJECT_TYPE=configmaps,cronjobs,daemonsets,deployment,endpoints,ingresses,jobs,networkpolicies,pods,podsecuritypolicies,persistentvolumeclaims,rolebindings,roles,secrets,serviceaccounts,services,statefulsets + export PARALLELISM_FACTOR=2 + function get_namespaces () { + kubectl get namespaces -o name | awk -F '/' '{ print $NF }' + } + + function list_namespaced_objects () { + export NAMESPACE=$1 + printf ${OBJECT_TYPE} | xargs -d ',' -I {} -P1 -n1 bash -c 'echo "${NAMESPACE} $@"' _ {} + } + export -f list_namespaced_objects + + function name_objects () { + input=($1) + export NAMESPACE=${input[0]} + export OBJECT=${input[1]} + kubectl get -n ${NAMESPACE} ${OBJECT} -o name | xargs -L1 -I {} -P1 -n1 bash -c 'echo "${NAMESPACE} ${OBJECT} $@"' _ {} + } + export -f name_objects + + function get_objects () { + input=($1) + export NAMESPACE=${input[0]} + export OBJECT=${input[1]} + export NAME=${input[2]#*/} + echo "${NAMESPACE}/${OBJECT}/${NAME}" + DIR="{{ logs_dir }}/objects/namespaced/${NAMESPACE}/${OBJECT}" + mkdir -p ${DIR} + kubectl get -n ${NAMESPACE} ${OBJECT} ${NAME} -o yaml > "${DIR}/${NAME}.yaml" + kubectl describe -n ${NAMESPACE} ${OBJECT} ${NAME} > "${DIR}/${NAME}.txt" + } + export -f get_objects + + get_namespaces | \ + xargs -r -n 1 -P ${PARALLELISM_FACTOR} -I {} bash -c 'list_namespaced_objects "$@"' _ {} | \ + xargs -r -n 1 -P ${PARALLELISM_FACTOR} -I {} bash -c 'name_objects "$@"' _ {} | \ + xargs -r -n 1 -P ${PARALLELISM_FACTOR} -I {} bash -c 'get_objects "$@"' _ {} + args: + executable: /bin/bash + ignore_errors: True + +- name: "Downloads logs to executor" + synchronize: + src: "{{ logs_dir }}/objects" + dest: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}" + mode: pull + ignore_errors: yes +... diff --git a/roles/disable-local-nameserver/tasks/main.yaml b/roles/disable-local-nameserver/tasks/main.yaml new file mode 100644 index 0000000000..f2ea4e91c4 --- /dev/null +++ b/roles/disable-local-nameserver/tasks/main.yaml @@ -0,0 +1,59 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# NOTE(portdirect): We disable the local nameserver as it interferes with the +# k8s dns-service and other local resolvers used for development use. +# See the following for the original config: +# * https://github.com/openstack/project-config/blob/0332c33dd134033e0620645c252f82b77e4c16f5/nodepool/elements/nodepool-base/finalise.d/89-unbound + +--- +- name: Disable local nameserver and systemd-resolved service + when: ansible_distribution == 'Ubuntu' + block: + - name: update rc.local + blockinfile: + path: /etc/rc.local + mode: 365 + block: | + #!/bin/bash + set -o xtrace + # Some providers inject dynamic network config statically. Work around this + # for DNS nameservers. This is expected to fail on some nodes so remove -e. + set +e + sed -i -e 's/^\(DNS[0-9]*=[.0-9]\+\)/#\1/g' /etc/sysconfig/network-scripts/ifcfg-* + sed -i -e 's/^NETCONFIG_DNS_POLICY=.*/NETCONFIG_DNS_POLICY=""/g' /etc/sysconfig/network/config + set -e + echo 'nameserver 208.67.222.222' > /etc/resolv.conf + echo 'nameserver 8.8.8.8' >> /etc/resolv.conf + exit 0 + - name: write resolv.conf + blockinfile: + path: /etc/resolv.conf + mode: 644 + block: | + nameserver 208.67.222.222 + nameserver 8.8.8.8 + - name: stop unbound service + systemd: + state: stopped + enabled: no + masked: yes + daemon_reload: yes + name: unbound + - name: stop systemd-resolved service + systemd: + state: stopped + enabled: no + masked: yes + daemon_reload: yes + name: systemd-resolved +... diff --git a/roles/enable-hugepages/defaults/main.yaml b/roles/enable-hugepages/defaults/main.yaml new file mode 100644 index 0000000000..cbdf0ae916 --- /dev/null +++ b/roles/enable-hugepages/defaults/main.yaml @@ -0,0 +1,21 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +hugepages: + enabled: false +# This parameter sets the size of the huge pages, available options: 2M and 1G + size: "2M" +# This parameter sets the number of huge pages to allocate + number: 1024 +grub_default_config: "/etc/default/grub" +... diff --git a/roles/enable-hugepages/tasks/main.yaml b/roles/enable-hugepages/tasks/main.yaml new file mode 100644 index 0000000000..605413cf7f --- /dev/null +++ b/roles/enable-hugepages/tasks/main.yaml @@ -0,0 +1,37 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Set up 1G hugepages + become: true + block: + - name: Configure grub + lineinfile: + dest: "{{ grub_default_config }}" + line: 'GRUB_CMDLINE_LINUX="default_hugepagesz={{ hugepages.size }} hugepagesz={{ hugepages.size }} hugepages={{ hugepages.number }}"' + regexp: '^GRUB_CMDLINE_LINUX="' + - name: Update grub configuration + command: update-grub2 + - name: Reboot host + reboot: + reboot_timeout: 600 + when: hugepages.size == "1G" + +- name: Set up 2M hugepages + become: true + sysctl: + name: vm.nr_hugepages + value: "{{ hugepages.number }}" + sysctl_set: true + reload: true + when: hugepages.size == "2M" +... diff --git a/roles/gather-host-logs/tasks/main.yaml b/roles/gather-host-logs/tasks/main.yaml new file mode 100644 index 0000000000..8031f9d68f --- /dev/null +++ b/roles/gather-host-logs/tasks/main.yaml @@ -0,0 +1,49 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: "creating directory for system status" + file: + path: "{{ logs_dir }}/system" + state: directory + +- name: "Get logs for each host" + become: yes + shell: |- + set -x + systemd-cgls --full --all --no-pager > {{ logs_dir }}/system/systemd-cgls.txt + ip addr > {{ logs_dir }}/system/ip-addr.txt + ip route > {{ logs_dir }}/system/ip-route.txt + lsblk > {{ logs_dir }}/system/lsblk.txt + mount > {{ logs_dir }}/system/mount.txt + docker images > {{ logs_dir }}/system/docker-images.txt + brctl show > {{ logs_dir }}/system/brctl-show.txt + ps aux --sort=-%mem > {{ logs_dir }}/system/ps.txt + dpkg -l > {{ logs_dir }}/system/packages.txt + CONTAINERS=($(docker ps -a --format {% raw %}'{{ .Names }}'{% endraw %} --filter label=zuul)) + if [ ! -z "$CONTAINERS" ]; then + mkdir -p "{{ logs_dir }}/system/containers" + for CONTAINER in ${CONTAINERS}; do + docker logs "${CONTAINER}" > "{{ logs_dir }}/system/containers/${CONTAINER}.txt" + done + fi + args: + executable: /bin/bash + ignore_errors: True + +- name: "Downloads logs to executor" + synchronize: + src: "{{ logs_dir }}/system" + dest: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}" + mode: pull + ignore_errors: True +... diff --git a/roles/gather-pod-logs/tasks/main.yaml b/roles/gather-pod-logs/tasks/main.yaml new file mode 100644 index 0000000000..d3a3a794ca --- /dev/null +++ b/roles/gather-pod-logs/tasks/main.yaml @@ -0,0 +1,63 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: "creating directory for pod logs" + file: + path: "{{ logs_dir }}/pod-logs" + state: directory + +- name: "creating directory for failed pod logs" + file: + path: "{{ logs_dir }}/pod-logs/failed-pods" + state: directory + +- name: "retrieve all kubernetes logs, current and previous (if they exist)" + shell: |- + set -e + PARALLELISM_FACTOR=2 + function get_namespaces () { + kubectl get namespaces -o name | awk -F '/' '{ print $NF }' + } + function get_pods () { + NAMESPACE=$1 + kubectl get pods -n ${NAMESPACE} -o name | awk -F '/' '{ print $NF }' | xargs -L1 -P 1 -I {} echo ${NAMESPACE} {} + } + export -f get_pods + function get_pod_logs () { + NAMESPACE=${1% *} + POD=${1#* } + INIT_CONTAINERS=$(kubectl get pod $POD -n ${NAMESPACE} -o json | jq -r '.spec.initContainers[]?.name') + CONTAINERS=$(kubectl get pod $POD -n ${NAMESPACE} -o json | jq -r '.spec.containers[].name') + for CONTAINER in ${INIT_CONTAINERS} ${CONTAINERS}; do + echo "${NAMESPACE}/${POD}/${CONTAINER}" + mkdir -p "{{ logs_dir }}/pod-logs/${NAMESPACE}/${POD}" + mkdir -p "{{ logs_dir }}/pod-logs/failed-pods/${NAMESPACE}/${POD}" + kubectl logs ${POD} -n ${NAMESPACE} -c ${CONTAINER} > "{{ logs_dir }}/pod-logs/${NAMESPACE}/${POD}/${CONTAINER}.txt" + kubectl logs --previous ${POD} -n ${NAMESPACE} -c ${CONTAINER} > "{{ logs_dir }}/pod-logs/failed-pods/${NAMESPACE}/${POD}/${CONTAINER}.txt" + done + } + export -f get_pod_logs + get_namespaces | \ + xargs -r -n 1 -P ${PARALLELISM_FACTOR} -I {} bash -c 'get_pods "$@"' _ {} | \ + xargs -r -n 2 -P ${PARALLELISM_FACTOR} -I {} bash -c 'get_pod_logs "$@"' _ {} + args: + executable: /bin/bash + ignore_errors: True + +- name: "Downloads pod logs to executor" + synchronize: + src: "{{ logs_dir }}/pod-logs" + dest: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}" + mode: pull + ignore_errors: True +... diff --git a/roles/gather-prom-metrics/tasks/main.yaml b/roles/gather-prom-metrics/tasks/main.yaml new file mode 100644 index 0000000000..30ea459526 --- /dev/null +++ b/roles/gather-prom-metrics/tasks/main.yaml @@ -0,0 +1,77 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: "creating directory for helm release descriptions" + file: + path: "{{ logs_dir }}/prometheus" + state: directory + +- name: "Get metrics from exporter services in all namespaces" + shell: |- + set -e + NAMESPACES=$(kubectl get namespaces -o json | jq -r '.items[].metadata.name') + for NS in $NAMESPACES; do + SERVICES=$(kubectl get svc -n $NS -o json | jq -r '.items[] | select(.spec.ports[].name=="metrics") | .metadata.name') + for SVC in $SERVICES; do + PORT=$(kubectl get svc $SVC -n $NS -o json | jq -r '.spec.ports[] | select(.name=="metrics") | .port') + echo "Scraping $SVC.$NS:$PORT/metrics:" + curl "$SVC.$NS:$PORT/metrics" >> "{{ logs_dir }}"/prometheus/$NS-$SVC.txt || true + done + done + args: + executable: /bin/bash + ignore_errors: True + +- name: "Get ceph metrics from ceph-mgr" + shell: |- + set -e + mgr_endpoints=$(kubectl get endpoints -n ceph -l component=manager -o json | jq -r '.items[].subsets[].addresses[].ip') + echo "ceph-mgr endpoints: $mgr_endpoints" + for endpoint in $mgr_endpoints; do + echo "checking ceph-mgr at $endpoint" + metrics_curl="curl $endpoint:9283/metrics" + op=$(eval "$metrics_curl") + if [[ -n $op ]]; then + curl $endpoint:9283/metrics >> "{{ logs_dir }}"/prometheus/ceph-ceph-mgr.txt + break + else + echo "$endpoint is a standby ceph-mgr. Trying next endpoint" + fi + done + args: + executable: /bin/bash + ignore_errors: True + +- name: "Get metrics from fluentd pods" + shell: |- + set -e + NAMESPACE="osh-infra" + APP_LABEL="fluentd" + PODS=$(kubectl get pods -n $NAMESPACE -l application=$APP_LABEL -o json | jq -r '.items[].metadata.name') + for POD in $PODS; do + IP=$(kubectl get pod -n $NAMESPACE $POD -o json | jq -r '.status.podIP') + PORT=$(kubectl get pod -n $NAMESPACE $POD -o json | jq -r '.spec.containers[0].ports[] | select(.name=="metrics") | .containerPort') + echo "Scraping $POD at $IP:$PORT/metrics" + curl "$IP:$PORT/metrics" >> "{{ logs_dir }}"/prometheus/$POD.txt || true + done + args: + executable: /bin/bash + ignore_errors: True + +- name: "Downloads logs to executor" + synchronize: + src: "{{ logs_dir }}/prometheus" + dest: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}" + mode: pull + ignore_errors: True +... diff --git a/roles/gather-selenium-data/tasks/main.yaml b/roles/gather-selenium-data/tasks/main.yaml new file mode 100644 index 0000000000..f5f32c199a --- /dev/null +++ b/roles/gather-selenium-data/tasks/main.yaml @@ -0,0 +1,33 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: "creating directory for helm release descriptions" + file: + path: "{{ logs_dir }}/selenium" + state: directory + +- name: "Get selenium data" + shell: |- + set -x + cp /tmp/artifacts/* {{ logs_dir }}/selenium/. + args: + executable: /bin/bash + ignore_errors: True + +- name: "Downloads logs to executor" + synchronize: + src: "{{ logs_dir }}/selenium" + dest: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}" + mode: pull + ignore_errors: True +... diff --git a/roles/helm-release-status/tasks/main.yaml b/roles/helm-release-status/tasks/main.yaml new file mode 100644 index 0000000000..35e199dad3 --- /dev/null +++ b/roles/helm-release-status/tasks/main.yaml @@ -0,0 +1,50 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: "creating directory for helm release status" + file: + path: "{{ logs_dir }}/helm/{{ directory }}" + state: directory + loop_control: + loop_var: directory + with_items: + - values + - releases + +- name: "Gather get release status for helm charts" + shell: |- + set -e + + for namespace in $(kubectl get namespaces --no-headers --output custom-columns=":metadata.name"); do + # get all Helm releases including pending and failed releases + for release in $(helm list --all --short --namespace $namespace); do + # Make respective directories only when a Helm release actually exists in the namespace + # to prevent uploading a bunch of empty directories for namespaces without a Helm release. + mkdir -p {{ logs_dir }}/helm/releases/$namespace + mkdir -p {{ logs_dir }}/helm/values/$namespace + + helm status $release --namespace $namespace >> {{ logs_dir }}/helm/releases/$namespace/$release.txt + helm get values $release --namespace $namespace --all >> {{ logs_dir }}/helm/values/$namespace/$release.yaml + done + done + args: + executable: /bin/bash + ignore_errors: True + +- name: "Downloads logs to executor" + synchronize: + src: "{{ logs_dir }}/helm" + dest: "{{ zuul.executor.log_root }}/{{ inventory_hostname }}" + mode: pull + ignore_errors: True +... diff --git a/roles/mount-extra-volume/defaults/main.yml b/roles/mount-extra-volume/defaults/main.yml new file mode 100644 index 0000000000..bdc745576c --- /dev/null +++ b/roles/mount-extra-volume/defaults/main.yml @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +extra_volume: + size: 80G + type: Linux + mount_point: /opt/ext_vol +... diff --git a/roles/mount-extra-volume/tasks/main.yaml b/roles/mount-extra-volume/tasks/main.yaml new file mode 100644 index 0000000000..f271bd9378 --- /dev/null +++ b/roles/mount-extra-volume/tasks/main.yaml @@ -0,0 +1,36 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Mount additional {{ extra_volume.size }} volume if available + when: + - ansible_distribution == 'Ubuntu' + - (ansible_mounts|selectattr("mount", "equalto", "/")|list)[0].size_available < 50000000000 + block: + - name: Mount additional {{ extra_volume.size }} volume if available + shell: | + set -ex + sudo fdisk --list + df -h + sudo mkdir -p ${EXTRA_VOLUME_MOUNT_POINT} + BIG_VOLUME=$(sudo fdisk -l 2>&1 | grep -E ${EXTRA_VOLUME_SIZE} | grep ${EXTRA_VOLUME_TYPE} | awk '{print $1}') + if ! mount | grep "${BIG_VOLUME}" + then + sudo mkfs.ext4 "${BIG_VOLUME}" + sudo mount "${BIG_VOLUME}" ${EXTRA_VOLUME_MOUNT_POINT} + df -h + fi + environment: + EXTRA_VOLUME_MOUNT_POINT: "{{ extra_volume.mount_point }}" + EXTRA_VOLUME_SIZE: "{{ extra_volume.size }}" + EXTRA_VOLUME_TYPE: "{{ extra_volume.type }}" +... diff --git a/roles/osh-bandit/defaults/main.yaml b/roles/osh-bandit/defaults/main.yaml new file mode 100644 index 0000000000..3d68528453 --- /dev/null +++ b/roles/osh-bandit/defaults/main.yaml @@ -0,0 +1,17 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +work_dir: "{{ zuul.project.src_dir }}" +helm_version: "v3.6.3" +bandit_version: "1.7.1" +... diff --git a/roles/osh-bandit/tasks/main.yaml b/roles/osh-bandit/tasks/main.yaml new file mode 100644 index 0000000000..961024b060 --- /dev/null +++ b/roles/osh-bandit/tasks/main.yaml @@ -0,0 +1,50 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Install Helm + shell: | + TMP_DIR=$(mktemp -d) + curl -sSL https://get.helm.sh/helm-{{ helm_version }}-linux-amd64.tar.gz | tar -zxv --strip-components=1 -C ${TMP_DIR} + mv "${TMP_DIR}"/helm /usr/local/bin/helm + rm -rf "${TMP_DIR}" + sudo -H pip3 install --upgrade yq bandit=={{ bandit_version }} setuptools + args: + chdir: "{{ work_dir }}" + +- name: Template out python files + shell: | + set -xe; + make all + mkdir -p python-files + EXCLUDES="helm-toolkit doc tests tools logs tmp roles playbooks releasenotes zuul.d python-files" + DIRS=`ls -d */ | cut -f1 -d'/'` + + for EX in $EXCLUDES; do + DIRS=`echo $DIRS | sed "s/\b$EX\b//g"` + done + + for DIR in $DIRS; do + PYFILES=$(helm template $DIR | yq 'select(.data != null) | .data | to_entries | map(select(.key | test(".*\\.py"))) | select(length > 0) | values[] | {(.key) : (.value)}' | jq -s add) + PYKEYS=$(echo "$PYFILES" | jq -r 'select(. != null) | keys[]') + for KEY in $PYKEYS; do + echo "$PYFILES" | jq -r --arg KEY "$KEY" '.[$KEY]' > ./python-files/"$DIR-$KEY" + done + done + args: + chdir: "{{ work_dir }}" + +- name: Run bandit against python files + shell: bandit -r ./python-files + args: + chdir: "{{ work_dir }}" +... diff --git a/roles/osh-run-script-set/defaults/main.yaml b/roles/osh-run-script-set/defaults/main.yaml new file mode 100644 index 0000000000..22e3eac497 --- /dev/null +++ b/roles/osh-run-script-set/defaults/main.yaml @@ -0,0 +1,22 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +ceph_osd_data_device: "/dev/loop0" +kubeadm: + pod_network_cidr: "10.244.0.0/16" +osh_params: + container_distro_name: ubuntu + container_distro_version: jammy +osh_values_overrides_path: "../openstack-helm/values_overrides" +osh_infra_values_overrides_path: "../openstack-helm-infra/values_overrides" +... diff --git a/roles/osh-run-script-set/tasks/main.yaml b/roles/osh-run-script-set/tasks/main.yaml new file mode 100644 index 0000000000..a6adec5438 --- /dev/null +++ b/roles/osh-run-script-set/tasks/main.yaml @@ -0,0 +1,64 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- block: + - name: "Run script set {{ workload }}" + shell: | + set -xe; + env + {{ gate_script_path }} + loop: "{{ workload }}" + loop_control: + loop_var: gate_script_path + pause: 5 + args: + chdir: "{{ zuul.project.src_dir }}/{{ gate_scripts_relative_path }}" + environment: + CEPH_OSD_DATA_DEVICE: "{{ ceph_osd_data_device }}" + POD_NETWORK_CIDR: "{{ kubeadm.pod_network_cidr }}" + zuul_site_mirror_fqdn: "{{ zuul_site_mirror_fqdn }}" + OSH_EXTRA_HELM_ARGS: "{{ zuul_osh_extra_helm_args_relative_path | default('') }}" + OSH_HELM_REPO: "{{ osh_helm_repo | default('../openstack-helm/') }}" + OSH_INFRA_HELM_REPO: "{{ osh_infra_helm_repo | default('../openstack-helm-infra/') }}" + DOWNLOAD_OVERRIDES: "{{ download_overrides | default('') }}" + OSH_PATH: "{{ zuul_osh_relative_path | default('../openstack-helm/') }}" + OSH_INFRA_PATH: "{{ zuul_osh_infra_relative_path | default('../openstack-helm-infra/') }}" + OSH_VALUES_OVERRIDES_PATH: "{{ osh_values_overrides_path }}" + OSH_INFRA_VALUES_OVERRIDES_PATH: "{{ osh_infra_values_overrides_path }}" + OPENSTACK_RELEASE: "{{ osh_params.openstack_release | default('') }}" + CONTAINER_DISTRO_NAME: "{{ osh_params.container_distro_name | default('') }}" + CONTAINER_DISTRO_VERSION: "{{ osh_params.container_distro_version | default('') }}" + FEATURES: "{{ osh_params.feature_gates | default('') | regex_replace(',', ' ') }} {{ osh_params.openstack_release | default('') }} {{ osh_params.container_distro_name | default('') }}_{{ osh_params.container_distro_version | default('') }} {{ osh_params.container_distro_name | default('') }}" + RUN_HELM_TESTS: "{{ run_helm_tests | default('yes') }}" + # NOTE(aostapenko) using bigger than async_status timeout due to async_status issue with + # not recognizing timed out jobs: https://github.com/ansible/ansible/issues/25637 + async: 3600 + poll: 0 + register: async_results + + - name: Wait for script set to finish + async_status: + jid: '{{ item.ansible_job_id }}' + register: jobs + until: jobs.finished + delay: 5 + retries: 360 + loop: "{{ async_results.results }}" + + always: + - name: Print script set output + shell: | + # NOTE(aostapenko) safely retrieving items for the unlikely case if jobs timed out in async_status + echo 'STDOUT:\n{{ item.get("stdout") | regex_replace("\'", "") }}\nSTDERR:\n{{ item.get("stderr") | regex_replace("\'", "") }}' + loop: "{{ jobs.results }}" +... diff --git a/roles/osh-run-script/defaults/main.yaml b/roles/osh-run-script/defaults/main.yaml new file mode 100644 index 0000000000..22e3eac497 --- /dev/null +++ b/roles/osh-run-script/defaults/main.yaml @@ -0,0 +1,22 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +ceph_osd_data_device: "/dev/loop0" +kubeadm: + pod_network_cidr: "10.244.0.0/16" +osh_params: + container_distro_name: ubuntu + container_distro_version: jammy +osh_values_overrides_path: "../openstack-helm/values_overrides" +osh_infra_values_overrides_path: "../openstack-helm-infra/values_overrides" +... diff --git a/roles/osh-run-script/tasks/main.yaml b/roles/osh-run-script/tasks/main.yaml new file mode 100644 index 0000000000..ba085fa168 --- /dev/null +++ b/roles/osh-run-script/tasks/main.yaml @@ -0,0 +1,40 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: "Run script {{ workload[0] }}" + shell: | + set -xe; + env + {{ gate_script_path }} + vars: + gate_script_path: "{{ workload[0] }}" + args: + chdir: "{{ zuul.project.src_dir }}/{{ gate_scripts_relative_path }}" + environment: + CEPH_OSD_DATA_DEVICE: "{{ ceph_osd_data_device }}" + POD_NETWORK_CIDR: "{{ kubeadm.pod_network_cidr }}" + zuul_site_mirror_fqdn: "{{ zuul_site_mirror_fqdn }}" + OSH_EXTRA_HELM_ARGS: "{{ zuul_osh_extra_helm_args_relative_path | default('') }}" + OSH_HELM_REPO: "{{ osh_helm_repo | default('../openstack-helm') }}" + OSH_INFRA_HELM_REPO: "{{ osh_infra_helm_repo | default('../openstack-helm-infra') }}" + DOWNLOAD_OVERRIDES: "{{ download_overrides | default('') }}" + OSH_PATH: "{{ zuul_osh_relative_path | default('../openstack-helm/') }}" + OSH_INFRA_PATH: "{{ zuul_osh_infra_relative_path | default('../openstack-helm-infra/') }}" + OSH_VALUES_OVERRIDES_PATH: "{{ osh_values_overrides_path }}" + OSH_INFRA_VALUES_OVERRIDES_PATH: "{{ osh_infra_values_overrides_path }}" + OPENSTACK_RELEASE: "{{ osh_params.openstack_release | default('') }}" + CONTAINER_DISTRO_NAME: "{{ osh_params.container_distro_name | default('') }}" + CONTAINER_DISTRO_VERSION: "{{ osh_params.container_distro_version | default('') }}" + FEATURES: "{{ osh_params.feature_gates | default('') | regex_replace(',', ' ') }} {{ osh_params.openstack_release | default('') }} {{ osh_params.container_distro_name | default('') }}_{{ osh_params.container_distro_version | default('') }} {{ osh_params.container_distro_name | default('') }}" + RUN_HELM_TESTS: "{{ run_helm_tests | default('yes') }}" +... diff --git a/roles/override-images/defaults/main.yaml b/roles/override-images/defaults/main.yaml new file mode 100644 index 0000000000..72d4fdbd4f --- /dev/null +++ b/roles/override-images/defaults/main.yaml @@ -0,0 +1,15 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +work_dir: "{{ zuul.project.src_dir }}" +... diff --git a/roles/override-images/tasks/main.yaml b/roles/override-images/tasks/main.yaml new file mode 100644 index 0000000000..566ce38e98 --- /dev/null +++ b/roles/override-images/tasks/main.yaml @@ -0,0 +1,48 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Use buildset registry + include_role: + name: use-buildset-registry + +- name: Print zuul + debug: + var: zuul + +- name: Override proposed images from artifacts + shell: > + find {{ override_paths | join(" ") }} -type f -exec sed -Ei + "s#['\"]?docker\.io/({{ repo }}):({{ tag }})['\"]?\$#{{ buildset_registry_alias }}:{{ buildset_registry.port }}/\1:\2#g" {} + + loop: "{{ zuul.artifacts | default([]) }}" + args: + chdir: "{{ work_dir }}" + loop_control: + loop_var: zj_zuul_artifact + when: "'metadata' in zj_zuul_artifact and zj_zuul_artifact.metadata.type | default('') == 'container_image'" + vars: + tag: "{{ zj_zuul_artifact.metadata.tag }}" + repo: "{{ zj_zuul_artifact.metadata.repository }}" + override_paths: + - ../openstack-helm*/*/values* + - ../openstack-helm-infra/tools/deployment/ + +- name: Diff + shell: | + set -ex; + for dir in openstack-helm openstack-helm-infra; do + path="{{ work_dir }}/../${dir}/" + if [ ! -d "${path}" ]; then continue; fi + echo "${dir} diff" + cd "${path}"; git diff; cd -; + done +... diff --git a/roles/setup-firewall/tasks/main.yaml b/roles/setup-firewall/tasks/main.yaml new file mode 100644 index 0000000000..64e75ddc70 --- /dev/null +++ b/roles/setup-firewall/tasks/main.yaml @@ -0,0 +1,29 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# NOTE(portdirect): This needs refinement but drops the firewall on zuul nodes +--- +- name: deploy iptables packages + include_role: + name: deploy-package + tasks_from: dist + vars: + packages: + deb: + - iptables + rpm: + - iptables +- command: iptables -S +- command: iptables -F +- command: iptables -P INPUT ACCEPT +- command: iptables -S +... diff --git a/roles/upgrade-host/defaults/main.yml b/roles/upgrade-host/defaults/main.yml new file mode 100644 index 0000000000..93b068cd78 --- /dev/null +++ b/roles/upgrade-host/defaults/main.yml @@ -0,0 +1,15 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +ubuntu_kernel_hwe: false +... diff --git a/roles/upgrade-host/tasks/main.yaml b/roles/upgrade-host/tasks/main.yaml new file mode 100644 index 0000000000..0afb373859 --- /dev/null +++ b/roles/upgrade-host/tasks/main.yaml @@ -0,0 +1,44 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +- name: Upgrade to HWE kernel on Ubuntu Hosts + when: + - ansible_distribution == 'Ubuntu' + - ubuntu_kernel_hwe == true + block: + - name: Deploy HWE kernel on Ubuntu Hosts + include_role: + name: deploy-package + tasks_from: dist + vars: + packages: + deb: + - linux-generic-hwe-16.04 + - name: Reboot Host following kernel upgrade + shell: sleep 2 && reboot + become: yes + async: 30 + poll: 0 + ignore_errors: true + args: + executable: /bin/bash + - name: Wait for hosts to come up following reboot + wait_for: + host: '{{ hostvars[item].ansible_host }}' + port: 22 + state: started + delay: 60 + timeout: 240 + with_items: '{{ play_hosts }}' + connection: local +... diff --git a/shaker/Chart.yaml b/shaker/Chart.yaml new file mode 100644 index 0000000000..dd01e4e2ef --- /dev/null +++ b/shaker/Chart.yaml @@ -0,0 +1,30 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +apiVersion: v2 +appVersion: v1.0.0 +description: OpenStack-Helm Shaker +name: shaker +version: 2024.2.0 +home: https://pyshaker.readthedocs.io/en/latest/index.html +icon: https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcTlnnEExfz6H9bBFFDxsDm5mVTdKWOt6Hw2_3aJ7hVkNdDdTCrimQ +sources: + - https://opendev.org/openstack/shaker + - https://opendev.org/openstack/openstack-helm +maintainers: + - name: OpenStack-Helm Authors +dependencies: + - name: helm-toolkit + repository: file://../helm-toolkit + version: ">= 0.1.0" +... diff --git a/shaker/templates/bin/_run-tests.sh.tpl b/shaker/templates/bin/_run-tests.sh.tpl new file mode 100644 index 0000000000..b8d23fa9a0 --- /dev/null +++ b/shaker/templates/bin/_run-tests.sh.tpl @@ -0,0 +1,19 @@ +#!/bin/bash + +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +set -ex + +{{ .Values.conf.script }} diff --git a/shaker/templates/configmap-bin.yaml b/shaker/templates/configmap-bin.yaml new file mode 100644 index 0000000000..371ce54973 --- /dev/null +++ b/shaker/templates/configmap-bin.yaml @@ -0,0 +1,32 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_bin }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: shaker-bin +data: +{{- if .Values.images.local_registry.active }} + image-repo-sync.sh: | +{{- include "helm-toolkit.scripts.image_repo_sync" . | indent 4 }} +{{- end }} + ks-user.sh: | +{{- include "helm-toolkit.scripts.keystone_user" . | indent 4 }} + run-tests.sh: | +{{ tuple "bin/_run-tests.sh.tpl" . | include "helm-toolkit.utils.template" | indent 4 }} +{{- end }} +... diff --git a/shaker/templates/configmap-etc.yaml b/shaker/templates/configmap-etc.yaml new file mode 100644 index 0000000000..0ec872e51e --- /dev/null +++ b/shaker/templates/configmap-etc.yaml @@ -0,0 +1,57 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.configmap_etc }} +{{- $envAll := . }} + +{{- if empty .Values.conf.shaker.auth.admin_username -}} +{{- $_ := set .Values.conf.shaker.auth "admin_username" .Values.endpoints.identity.auth.admin.username -}} +{{- end -}} +{{- if empty .Values.conf.shaker.auth.admin_password -}} +{{- $_ := set .Values.conf.shaker.auth "admin_password" .Values.endpoints.identity.auth.admin.password -}} +{{- end -}} +{{- if empty .Values.conf.shaker.auth.admin_project_name -}} +{{- $_ := set .Values.conf.shaker.auth "admin_project_name" .Values.endpoints.identity.auth.admin.project_name -}} +{{- end -}} +{{- if empty .Values.conf.shaker.auth.admin_domain_name -}} +{{- $_ := set .Values.conf.shaker.auth "admin_domain_name" .Values.endpoints.identity.auth.admin.user_domain_name -}} +{{- end -}} +{{- if empty .Values.conf.shaker.auth.admin_domain_scope -}} +{{- $_ := set .Values.conf.shaker.auth "admin_domain_scope" .Values.endpoints.identity.auth.admin.user_domain_name -}} +{{- end -}} + +{{- if empty .Values.conf.shaker.identity.uri_v3 -}} +{{- $_ := tuple "identity" "internal" "api" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup"| set .Values.conf.shaker.identity "uri_v3" -}} +{{- end -}} + +{{- if empty .Values.conf.shaker.identity.region -}} +{{- $_ := set .Values.conf.shaker.identity "region" .Values.endpoints.identity.auth.admin.region_name -}} +{{- end -}} + +--- +apiVersion: v1 +kind: Secret +metadata: + name: shaker-etc +type: Opaque +data: + shaker.conf: {{ include "helm-toolkit.utils.to_oslo_conf" .Values.conf.shaker.shaker | b64enc }} +{{ if not (empty .Values.conf.basic) }} + test-basic: {{ include "shaker.utils.to_regex_file" .Values.conf.basic | b64enc }} +{{ end }} +{{ if not (empty .Values.conf.sriov) }} + test-sriov: {{ include "shaker.utils.to_regex_file" .Values.conf.sriov | b64enc }} +{{ end }} +{{- end }} +... diff --git a/shaker/templates/job-image-repo-sync.yaml b/shaker/templates/job-image-repo-sync.yaml new file mode 100644 index 0000000000..12738d9421 --- /dev/null +++ b/shaker/templates/job-image-repo-sync.yaml @@ -0,0 +1,19 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.job_image_repo_sync .Values.images.local_registry.active }} +{{- $imageRepoSyncJob := dict "envAll" . "serviceName" "shaker" -}} +{{ $imageRepoSyncJob | include "helm-toolkit.manifests.job_image_repo_sync" }} +{{- end }} + diff --git a/shaker/templates/job-ks-user.yaml b/shaker/templates/job-ks-user.yaml new file mode 100644 index 0000000000..94be5bd59b --- /dev/null +++ b/shaker/templates/job-ks-user.yaml @@ -0,0 +1,19 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.job_ks_user }} +{{- $ksUserJob := dict "envAll" . "serviceName" "shaker" -}} +{{ $ksUserJob | include "helm-toolkit.manifests.job_ks_user" }} +{{- end }} +... diff --git a/shaker/templates/pod-shaker-test.yaml b/shaker/templates/pod-shaker-test.yaml new file mode 100644 index 0000000000..b4fe18d863 --- /dev/null +++ b/shaker/templates/pod-shaker-test.yaml @@ -0,0 +1,141 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.pod_shaker_test }} +{{- $envAll := . }} + +{{- $mounts_tests := .Values.pod.mounts.shaker_tests.shaker_tests }} +{{- $mounts_tests_init := .Values.pod.mounts.shaker_tests.init_container }} + +{{- $serviceAccountName := print $envAll.Release.Name "-test" }} +{{ tuple $envAll "run_tests" $serviceAccountName | include "helm-toolkit.snippets.kubernetes_pod_rbac_serviceaccount" }} +--- +apiVersion: v1 +kind: Pod +metadata: + name: {{ print $envAll.Release.Name "-run-tests" }} + labels: +{{ tuple $envAll "shaker" "run-tests" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + annotations: + "helm.sh/hook": test-success +spec: + nodeSelector: + {{ .Values.labels.pod.node_selector_key }}: {{ .Values.labels.pod.node_selector_value }} + serviceAccountName: {{ $serviceAccountName }} + restartPolicy: OnFailure + initContainers: +{{ tuple $envAll "run_tests" $mounts_tests_init | include "helm-toolkit.snippets.kubernetes_entrypoint_init_container" | indent 4 }} + - name: {{ .Release.Name }}-test-ks-user +{{ tuple $envAll "ks_user" | include "helm-toolkit.snippets.image" | indent 6 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.ks_user | include "helm-toolkit.snippets.kubernetes_resources" | indent 6 }} + command: + - /tmp/ks-user.sh + volumeMounts: + - name: shaker-bin + mountPath: /tmp/ks-user.sh + subPath: ks-user.sh + readOnly: true + env: +{{- with $env := dict "ksUserSecret" .Values.secrets.identity.admin }} +{{- include "helm-toolkit.snippets.keystone_openrc_env_vars" $env | indent 8 }} +{{- end }} + - name: SERVICE_OS_SERVICE_NAME + value: "shaker" +{{- with $env := dict "ksUserSecret" .Values.secrets.identity.shaker }} +{{- include "helm-toolkit.snippets.keystone_user_create_env_vars" $env | indent 8 }} +{{- end }} + - name: SERVICE_OS_ROLE + value: {{ .Values.endpoints.identity.auth.shaker.role | quote }} + - name: {{ .Release.Name }}-perms +{{ tuple $envAll "shaker_run_tests" | include "helm-toolkit.snippets.image" | indent 6 }} + securityContext: + runAsUser: 0 + privileged: true +{{ tuple $envAll $envAll.Values.pod.resources.jobs.run_tests | include "helm-toolkit.snippets.kubernetes_resources" | indent 6 }} + command: ["/bin/sh", "-c"] + args: + - set -xe; + chmod 0777 /opt/shaker/data/; + chmod 0777 /opt/shaker-data/; + volumeMounts: + - name: shaker-reports + mountPath: /opt/shaker/data/ + - name: shaker-data-host + mountPath: /opt/shaker-data/ + containers: + - name: {{ .Release.Name }}-run-tests +{{ tuple $envAll "shaker_run_tests" | include "helm-toolkit.snippets.image" | indent 6 }} +{{ tuple $envAll $envAll.Values.pod.resources.jobs.run_tests | include "helm-toolkit.snippets.kubernetes_resources" | indent 6 }} + securityContext: + runAsUser: {{ .Values.pod.user.shaker.uid }} + privileged: false + env: +{{- with $env := dict "ksUserSecret" .Values.secrets.identity.admin }} +{{- include "helm-toolkit.snippets.keystone_openrc_env_vars" $env | indent 8 }} +{{- end }} +{{- with $env := dict "ksUserSecret" .Values.secrets.identity.shaker }} +{{- include "helm-toolkit.snippets.keystone_user_create_env_vars" $env | indent 8 }} +{{- end }} + - name: SHAKER_ENV_NAME + value: {{.Release.Name}} + - name: SHAKER_SCENARIO + value: {{ .Values.conf.shaker.shaker.DEFAULT.scenario }} + - name: SHAKER_SERVER_ENDPOINT + value: {{ .Values.conf.shaker.shaker.DEFAULT.server_endpoint }} + command: + - /tmp/run-tests.sh + volumeMounts: + - name: shaker-etc + mountPath: /etc/shaker/shaker_tests.yaml + subPath: shaker_tests.yaml + readOnly: true + - name: shaker-bin + mountPath: /tmp/run-tests.sh + subPath: run-tests.sh + readOnly: true + - name: shaker-db + mountPath: /opt/shaker/db/ + - name: shaker-reports + mountPath: /opt/shaker/data/ + - name: shaker-data-host + mountPath: /opt/shaker-data/ + - name: shaker-etc + mountPath: /opt/shaker/shaker.conf + subPath: shaker.conf + readOnly: true +{{ if $mounts_tests.volumeMounts }}{{ toYaml $mounts_tests.volumeMounts | indent 8 }}{{ end }} + volumes: + - name: shaker-etc + secret: + secretName: shaker-etc + defaultMode: 0444 + - name: shaker-bin + configMap: + name: shaker-bin + defaultMode: 0555 + - name: shaker-db + emptyDir: {} + - name: shaker-reports + {{- if not .Values.pvc.enabled }} + emptyDir: {} + {{- else }} + persistentVolumeClaim: + claimName: {{ .Values.pvc.name }} + {{- end }} + - name: shaker-data-host + hostPath: + path: /tmp/shaker-data +{{ if $mounts_tests.volumes }}{{ toYaml $mounts_tests.volumes | indent 4 }}{{ end }} +{{- end }} +... diff --git a/shaker/templates/pvc-shaker.yaml b/shaker/templates/pvc-shaker.yaml new file mode 100644 index 0000000000..fbc03d7620 --- /dev/null +++ b/shaker/templates/pvc-shaker.yaml @@ -0,0 +1,29 @@ +# {{/* +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# */}} + +{{- if .Values.pvc.enabled }} + +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: {{ .Values.pvc.name }} +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ .Values.pvc.requests.storage }} + storageClassName: {{ .Values.pvc.storage_class }} +{{- end }} +... diff --git a/shaker/templates/secret-keystone.yaml b/shaker/templates/secret-keystone.yaml new file mode 100644 index 0000000000..a9a0c126c5 --- /dev/null +++ b/shaker/templates/secret-keystone.yaml @@ -0,0 +1,29 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.secret_keystone }} +{{- $envAll := . }} +{{- range $key1, $userClass := tuple "admin" "shaker" }} +{{- $secretName := index $envAll.Values.secrets.identity $userClass }} +--- +apiVersion: v1 +kind: Secret +metadata: + name: {{ $secretName }} +type: Opaque +data: +{{- tuple $userClass "internal" $envAll | include "helm-toolkit.snippets.keystone_secret_openrc" | indent 2 -}} +{{- end }} +{{- end }} +... diff --git a/shaker/templates/secret-registry.yaml b/shaker/templates/secret-registry.yaml new file mode 100644 index 0000000000..da979b3223 --- /dev/null +++ b/shaker/templates/secret-registry.yaml @@ -0,0 +1,17 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if and .Values.manifests.secret_registry .Values.endpoints.oci_image_registry.auth.enabled }} +{{ include "helm-toolkit.manifests.secret_registry" ( dict "envAll" . "registryUser" .Chart.Name ) }} +{{- end }} diff --git a/shaker/templates/service-shaker.yaml b/shaker/templates/service-shaker.yaml new file mode 100644 index 0000000000..8d4fecfa49 --- /dev/null +++ b/shaker/templates/service-shaker.yaml @@ -0,0 +1,42 @@ +{{/* +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/}} + +{{- if .Values.manifests.service_shaker }} +{{- $envAll := . }} +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ tuple "shaker" "internal" . | include "helm-toolkit.endpoints.hostname_short_endpoint_lookup" }} +spec: + ports: + - name: shaker-api + protocol: TCP + port: {{ tuple "shaker" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + {{ if .Values.shaker.controller.node_port.enabled }} + nodePort: {{ .Values.shaker.controller.node_port.port }} + {{ end }} + targetPort: {{ tuple "shaker" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + selector: +{{ tuple $envAll "shaker" "run-tests" | include "helm-toolkit.snippets.kubernetes_metadata_labels" | indent 4 }} + {{ if .Values.shaker.controller.node_port.enabled }} + type: NodePort + {{ if .Values.shaker.controller.external_policy_local }} + externalTrafficPolicy: Local + {{ end }} + {{ end }} + externalIPs: + - {{ .Values.shaker.controller.external_ip }} +{{- end }} +... diff --git a/shaker/values.yaml b/shaker/values.yaml new file mode 100644 index 0000000000..472d7ab0e2 --- /dev/null +++ b/shaker/values.yaml @@ -0,0 +1,269 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Default values for shaker. +# This is a YAML-formatted file. +# Declare name/value pairs to be passed into your templates. +# name: value + +--- +labels: + job: + node_selector_key: openstack-control-plane + node_selector_value: enabled + pod: + node_selector_key: openstack-control-plane + node_selector_value: enabled + +images: + tags: + dep_check: quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal + shaker_run_tests: docker.io/performa/shaker:latest + ks_user: docker.io/openstackhelm/heat:wallaby-ubuntu_focal + image_repo_sync: docker.io/library/docker:17.07.0 + pull_policy: "IfNotPresent" + local_registry: + active: false + exclude: + - dep_check + - image_repo_sync + +pod: + user: + shaker: + uid: 1000 + resources: + enabled: false + jobs: + ks_user: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + run_tests: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + image_repo_sync: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "1024Mi" + cpu: "2000m" + mounts: + shaker_tests: + init_container: null + shaker_tests: + +shaker: + controller: + ingress: + public: true + classes: + namespace: "nginx" + cluster: "nginx-cluster" + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / + external_policy_local: false + node_port: + enabled: true + port: 31999 + external_ip: 9.9.9.9 + +dependencies: + dynamic: + common: + local_image_registry: + jobs: + - shaker-image-repo-sync + services: + - endpoint: node + service: local_image_registry + static: + ks_user: + services: + - service: identity + endpoint: internal + run_tests: + jobs: + - shaker-ks-user + services: + - service: identity + endpoint: internal + image_repo_sync: + services: + - endpoint: internal + service: local_image_registry + +conf: + script: | + sed -i -E "s/(accommodation\: \[.+)(.+\])/accommodation\: \[pair, compute_nodes: 1\]/" /opt/shaker/shaker/scenarios/openstack/full_l2.yaml + export server_endpoint=\`ip a | grep "global eth0" | cut -f6 -d' ' | cut -f1 -d'/'\` + + echo ========== SHAKER CONF PARAMETERS ================= + cat /opt/shaker/shaker.conf + echo ===================================================== + + env -i HOME="$HOME" bash -l -c "printenv; shaker --server-endpoint \$server_endpoint:31999 --config-file /opt/shaker/shaker.conf" + + shaker: + auth: + use_dynamic_credentials: true + admin_domain_scope: true + shaker_roles: admin, member + min_compute_nodes: 1 + identity: + auth_version: v3 + identity-feature-enabled: + api_v2: false + api_v3: true + shaker: + DEFAULT: + debug: true + cleanup_on_error: true + scenario_compute_nodes: 1 + report: /opt/shaker/data/shaker-result.html + output: /opt/shaker/data/shaker-result.json + scenario: /opt/shaker/shaker/scenarios/openstack/full_l2.yaml + flavor_name: m1.small + external_net: public + image_name: shaker-image + scenario_availability_zone: nova + os_username: admin + os_password: password + os_auth_url: "http://keystone.openstack.svc.cluster.local/v3" + os_project_name: admin + os_region_name: RegionOne + os_identity_api_version: 3 + os_interface: public + validation: + connect_method: floating + volume: + disk_formats: raw + backend_name: rbd1 + storage_protocol: rbd + volume-feature-enabled: + api_v1: False + api_v3: True + +pvc: + enabled: true + name: pvc-shaker + requests: + storage: 2Gi + storage_class: general + +secrets: + identity: + admin: shaker-keystone-admin + shaker: shaker-keystone-user + oci_image_registry: + shaker: shaker-oci-image-registry-key + +endpoints: + cluster_domain_suffix: cluster.local + local_image_registry: + name: docker-registry + namespace: docker-registry + hosts: + default: localhost + internal: docker-registry + node: localhost + host_fqdn_override: + default: null + port: + registry: + node: 5000 + oci_image_registry: + name: oci-image-registry + namespace: oci-image-registry + auth: + enabled: false + shaker: + username: shaker + password: password + hosts: + default: localhost + host_fqdn_override: + default: null + port: + registry: + default: null + identity: + name: keystone + auth: + admin: + region_name: RegionOne + username: admin + password: password + project_name: admin + user_domain_name: default + project_domain_name: default + shaker: + role: admin + region_name: RegionOne + username: shaker + password: password + project_name: service + user_domain_name: service + project_domain_name: service + hosts: + default: keystone + internal: keystone-api + host_fqdn_override: + default: null + path: + default: /v3 + scheme: + default: http + port: + api: + default: 80 + internal: 5000 + shaker: + name: shaker + hosts: + default: shaker + public: shaker + host_fqdn_override: + default: null + # NOTE(portdirect): this chart supports TLS for fqdn over-ridden public + # endpoints using the following format: + # public: + # host: null + # tls: + # crt: null + # key: null + path: + default: null + scheme: + default: 'http' + port: + api: + default: 31999 + public: 80 +manifests: + configmap_bin: true + configmap_etc: true + job_image_repo_sync: true + job_ks_user: true + pod_shaker_test: true + service_shaker: true + secret_keystone: true + secret_registry: true +... diff --git a/tools/deployment/ceph/ceph-adapter-rook.sh b/tools/deployment/ceph/ceph-adapter-rook.sh new file mode 100755 index 0000000000..bdf2bd0c92 --- /dev/null +++ b/tools/deployment/ceph/ceph-adapter-rook.sh @@ -0,0 +1,26 @@ + +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +#NOTE: Define variables +: ${OSH_INFRA_HELM_REPO:="../openstack-helm-infra"} +: ${OSH_INFRA_VALUES_OVERRIDES_PATH:="../openstack-helm-infra/values_overrides"} + +helm upgrade --install ceph-adapter-rook ${OSH_INFRA_HELM_REPO}/ceph-adapter-rook \ + --namespace=openstack + +#NOTE: Wait for deploy +helm osh wait-for-pods openstack diff --git a/tools/deployment/ceph/ceph-ns-activate.sh b/tools/deployment/ceph/ceph-ns-activate.sh new file mode 100755 index 0000000000..4e7bd33b66 --- /dev/null +++ b/tools/deployment/ceph/ceph-ns-activate.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +: ${OSH_INFRA_HELM_REPO:="../openstack-helm-infra"} +: ${OSH_INFRA_VALUES_OVERRIDES_PATH:="../openstack-helm-infra/values_overrides"} + +#NOTE: Deploy command +tee /tmp/ceph-openstack-config.yaml < /tmp/ceph-fs-uuid.txt +CEPH_FS_ID="$(cat /tmp/ceph-fs-uuid.txt)" +#NOTE(portdirect): to use RBD devices with Ubuntu kernels < 4.5 this +# should be set to 'hammer' +. /etc/os-release +if [ "x${ID}" == "xcentos" ] || \ + ([ "x${ID}" == "xubuntu" ] && \ + dpkg --compare-versions "$(uname -r)" "lt" "4.5"); then + CRUSH_TUNABLES=hammer +else + CRUSH_TUNABLES=null +fi +tee /tmp/rook.yaml < /dev/null 2>&1; then + kubectl wait --namespace=ceph --for=condition=ready "pod/$MON_POD" --timeout=600s + else + echo "Pod $MON_POD not found, skipping..." + fi +done + +echo "=========== CEPH K8S PODS LIST ============" +kubectl get pods -n rook-ceph -o wide +kubectl get pods -n ceph -o wide +#NOTE: Wait for deploy +RGW_POD=$(kubectl get pods \ + --namespace=ceph \ + --selector="app=rook-ceph-rgw" \ + --no-headers | awk '{print $1; exit}') +while [[ -z "${RGW_POD}" ]] +do + sleep 10 + date +'%Y-%m-%d %H:%M:%S' + TOOLS_POD=$(kubectl get pods \ + --namespace=ceph \ + --selector="app=rook-ceph-tools" \ + --no-headers | grep Running | awk '{ print $1; exit }') + if [[ -z "${TOOLS_POD}" ]]; then + echo "No running rook-ceph-tools pod found. Waiting..." + continue + fi + echo "=========== CEPH STATUS ============" + kubectl exec -n ceph ${TOOLS_POD} -- ceph -s + echo "=========== CEPH OSD POOL LIST ============" + kubectl exec -n ceph ${TOOLS_POD} -- ceph osd pool ls + echo "=========== CEPH K8S PODS LIST ============" + kubectl get pods -n ceph -o wide + RGW_POD=$(kubectl get pods \ + --namespace=ceph \ + --selector="app=rook-ceph-rgw" \ + --no-headers | awk '{print $1; exit}') +done +helm osh wait-for-pods ceph + +#NOTE: Validate deploy +TOOLS_POD=$(kubectl get pods \ + --namespace=ceph \ + --selector="app=rook-ceph-tools" \ + --no-headers | grep Running | awk '{ print $1; exit }') +kubectl exec -n ceph ${TOOLS_POD} -- ceph -s diff --git a/tools/deployment/ceph/ceph.sh b/tools/deployment/ceph/ceph.sh new file mode 100755 index 0000000000..5ee78584ef --- /dev/null +++ b/tools/deployment/ceph/ceph.sh @@ -0,0 +1,225 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +: ${CEPH_OSD_DATA_DEVICE:="/dev/loop100"} +: ${POD_NETWORK_CIDR:="10.244.0.0/16"} +: ${OSH_INFRA_HELM_REPO:="../openstack-helm-infra"} +: ${OSH_INFRA_VALUES_OVERRIDES_PATH:="../openstack-helm-infra/values_overrides"} + +NUMBER_OF_OSDS="$(kubectl get nodes -l ceph-osd=enabled --no-headers | wc -l)" + +#NOTE: Deploy command +[ -s /tmp/ceph-fs-uuid.txt ] || uuidgen > /tmp/ceph-fs-uuid.txt +CEPH_FS_ID="$(cat /tmp/ceph-fs-uuid.txt)" +#NOTE(portdirect): to use RBD devices with Ubuntu kernels < 4.5 this +# should be set to 'hammer' +. /etc/os-release +if [ "x${ID}" == "xcentos" ] || \ + ([ "x${ID}" == "xubuntu" ] && \ + dpkg --compare-versions "$(uname -r)" "lt" "4.5"); then + CRUSH_TUNABLES=hammer +else + CRUSH_TUNABLES=null +fi +tee /tmp/ceph.yaml < /tmp/ceph-fs-uuid.txt +CEPH_FS_ID="$(cat /tmp/ceph-fs-uuid.txt)" +#NOTE(portdirect): to use RBD devices with Ubuntu kernels < 4.5 this +# should be set to 'hammer' +. /etc/os-release +if [ "x${ID}" == "xcentos" ] || \ + ([ "x${ID}" == "xubuntu" ] && \ + dpkg --compare-versions "$(uname -r)" "lt" "4.5"); then + CRUSH_TUNABLES=hammer +else + CRUSH_TUNABLES=null +fi + +# Most of PV fields are immutable and in case of CSI RBD plugin they refer +# to secrets which were used for RBD provisioner and RBD attacher. These fields +# can not be updated later. +# So for testing purposes we assume legacy Ceph cluster is deployed with +# the following secret names for the CSI plugin +# - rook-csi-rbd-provisioner +# - rook-csi-rbd-node +# These exact secret names are used by Rook by default for CSI plugin and +# and after migration PVs will be adopted by the new Rook Ceph cluster. +# +# Alternatively if we deploy legacy Ceph cluster with the default values +# then we could later force Rook to use same CSI secret names as used for +# legacy cluster. For example pvc-ceph-conf-combined-storageclass secret +# name is used by default in legacy charts. +# +# Same is for CSI provisioner drivername option. For testing we deploy +# legacy cluster with the drivername set to rook-ceph.rbd.csi.ceph.com +# while default value is ceph.rbd.csi.ceph.com. +# This is also for the sake of smooth adoption of PVs. + +tee /tmp/ceph.yaml < /tmp/mon-a.keyring +export MON_HOST=$(kubectl -n ${CEPH_NAMESPACE} get pods -l app=rook-ceph-mon -o json | jq -r '.items[0].spec.nodeName') +export MON_HOST_IP=$(kubectl get nodes -o json | jq -r '.items[] | select(.metadata.name == env.MON_HOST) | .status.addresses | .[] | select(.type == "InternalIP") | .address') + +# Shut down the Rook operator, delete the rook-ceph deployments, and get the new rook-ceph-mon IP address +kubectl -n ${ROOK_CEPH_NAMESPACE} scale deploy rook-ceph-operator --replicas=0 +kubectl -n ${CEPH_NAMESPACE} get deploy -o json | jq -r '.items[] | select(.metadata.name != "rook-ceph-tools") | .metadata.name' | xargs kubectl -n ${CEPH_NAMESPACE} delete deploy +#MON_IP=$(kubectl -n ${CEPH_NAMESPACE} get service rook-ceph-mon-a -o json | jq -r '.spec.clusterIP') +MON_IP=$(kubectl -n ${CEPH_NAMESPACE} get cm rook-ceph-mon-endpoints -o jsonpath='{.data.data}' | sed 's/.=//g' | awk -F: '{print $1}') +wait_for_terminate + +# Download the old mon store and update its key to the new one +ssh ${MON_HOST_IP} "sudo rm -rf /var/lib/rook/mon-a/data" +ssh ${OLD_MON_HOST_IP} "sudo chmod -R a+rX /var/lib/openstack-helm/ceph/mon/mon/ceph-${OLD_MON_HOST}" +scp -rp ${OLD_MON_HOST_IP}:/var/lib/openstack-helm/ceph/mon/mon/ceph-${OLD_MON_HOST} /tmp +mv /tmp/ceph-${OLD_MON_HOST} /tmp/mon-a +grep -A2 "\[mon\.\]" /tmp/mon-a.keyring > /tmp/mon-a/keyring + +# Generate a script to rewrite the monmap in the old mon store +cat > /tmp/mon-a/fix-monmap.sh < /tmp/keyring" +kubectl -n ${CEPH_NAMESPACE} exec ${TOOLS_POD} -- bash -c "echo -e \" key = ${CLIENT_KEY}\" >> /tmp/keyring" +kubectl -n ${CEPH_NAMESPACE} exec ${TOOLS_POD} -- bash -c "echo -e ' caps mds = \"allow *\"' >> /tmp/keyring" +kubectl -n ${CEPH_NAMESPACE} exec ${TOOLS_POD} -- bash -c "echo -e ' caps mon = \"allow *\"' >> /tmp/keyring" +kubectl -n ${CEPH_NAMESPACE} exec ${TOOLS_POD} -- bash -c "echo -e ' caps osd = \"allow *\"' >> /tmp/keyring" +kubectl -n ${CEPH_NAMESPACE} exec ${TOOLS_POD} -- bash -c "echo -e ' caps mgr = \"allow *\"' >> /tmp/keyring" +kubectl -n ${CEPH_NAMESPACE} exec ${TOOLS_POD} -- ceph auth import -i /tmp/keyring +kubectl -n ${CEPH_NAMESPACE} exec ${TOOLS_POD} -- rm /tmp/keyring + +# Remove the auth config options to re-enable authentication +kubectl -n ${CEPH_NAMESPACE} get cm rook-config-override -o yaml | \ +sed '/ auth_cluster_required = none/d' | \ +sed '/ auth_service_required = none/d' | \ +sed '/ auth_client_required = none/d' | \ +sed '/ auth_supported = none/d' | \ +kubectl apply -f - + +# Restart the Rook operator and Ceph cluster with the new config +kubectl -n ${ROOK_CEPH_NAMESPACE} scale deploy rook-ceph-operator --replicas=0 +kubectl -n ${CEPH_NAMESPACE} get deploy -o json | jq -r '.items[] | select(.metadata.name != "rook-ceph-tools") | .metadata.name' | xargs kubectl -n ${CEPH_NAMESPACE} delete deploy +wait_for_terminate +kubectl -n ${ROOK_CEPH_NAMESPACE} scale deploy rook-ceph-operator --replicas=1 +wait_for_full_rook_deployment + +# Scale the mon and mgr deployments to original replica counts +kubectl -n ${CEPH_NAMESPACE} get cephcluster ceph -o json | \ +jq ".spec.mon.count = ${MON_COUNT} | .spec.mgr.count = ${MGR_COUNT}" | \ +kubectl apply -f - +wait_for_health_checks ${CEPH_NAMESPACE} ${TOOLS_POD} diff --git a/tools/deployment/ceph/migrate-values.sh b/tools/deployment/ceph/migrate-values.sh new file mode 100755 index 0000000000..e81ee444ac --- /dev/null +++ b/tools/deployment/ceph/migrate-values.sh @@ -0,0 +1,621 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +ROOK_RELEASE=v1.16.3 + +: ${CEPH_OSD_DATA_DEVICE:="/dev/loop100"} + +tee /tmp/rook-operator.yaml <&1)" =~ NotFound ]] || [ $NEXT_WAIT_TIME -eq 5 ]; do + sleep 20 + NEXT_WAIT_TIME=$((NEXT_WAIT_TIME+1)) +done diff --git a/tools/deployment/common/deploy-docker-registry.sh b/tools/deployment/common/deploy-docker-registry.sh new file mode 100755 index 0000000000..2cb5f98e17 --- /dev/null +++ b/tools/deployment/common/deploy-docker-registry.sh @@ -0,0 +1,69 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +: ${OSH_INFRA_HELM_REPO:="../openstack-helm-infra"} + +for NAMESPACE in docker-nfs docker-registry; do +tee /tmp/${NAMESPACE}-ns.yaml << EOF +apiVersion: v1 +kind: Namespace +metadata: + labels: + kubernetes.io/metadata.name: ${NAMESPACE} + name: ${NAMESPACE} + name: ${NAMESPACE} +EOF + +kubectl apply -f /tmp/${NAMESPACE}-ns.yaml +done + +#NOTE: Deploy nfs for the docker registry +tee /tmp/docker-registry-nfs-provisioner.yaml << EOF +labels: + node_selector_key: openstack-helm-node-class + node_selector_value: primary +storageclass: + name: openstack-helm-bootstrap +EOF +helm upgrade --install docker-registry-nfs-provisioner \ + ${OSH_INFRA_HELM_REPO}/nfs-provisioner --namespace=docker-nfs \ + --values=/tmp/docker-registry-nfs-provisioner.yaml + +#NOTE: Deploy redis for the docker registry +helm upgrade --install docker-registry-redis ${OSH_INFRA_HELM_REPO}/redis \ + --namespace=docker-registry \ + --set labels.node_selector_key=openstack-helm-node-class \ + --set labels.node_selector_value=primary + +#NOTE: Deploy the docker registry +tee /tmp/docker-registry.yaml << EOF +labels: + node_selector_key: openstack-helm-node-class + node_selector_value: primary +volume: + class_name: openstack-helm-bootstrap +EOF +helm upgrade --install docker-registry ${OSH_INFRA_HELM_REPO}/registry \ + --namespace=docker-registry \ + --values=/tmp/docker-registry.yaml + +#NOTE: Wait for deployments +helm osh wait-for-pods docker-registry + +# Delete the test pod if it still exists +kubectl delete pods -l application=redis,release_group=docker-registry-redis,component=test --namespace=docker-registry --ignore-not-found +#NOTE: Run helm tests +helm test docker-registry-redis --namespace docker-registry diff --git a/tools/deployment/common/falco.sh b/tools/deployment/common/falco.sh new file mode 100755 index 0000000000..8ed9ad025f --- /dev/null +++ b/tools/deployment/common/falco.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +: ${OSH_INFRA_HELM_REPO:="../openstack-helm-infra"} + +#NOTE: Deploy command +helm upgrade --install falco ${OSH_INFRA_HELM_REPO}/falco \ + --namespace=kube-system + +#NOTE: Wait for deploy +helm osh wait-for-pods kube-system diff --git a/tools/deployment/common/infra-prepare-charts.sh b/tools/deployment/common/infra-prepare-charts.sh new file mode 100755 index 0000000000..ee25bd326e --- /dev/null +++ b/tools/deployment/common/infra-prepare-charts.sh @@ -0,0 +1,23 @@ +#!/bin/bash +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -ex + +# Build all OSH charts +make all SKIP_CHANGELOG=1 + +# Build all OSH charts (necessary for Openstack deployment) +( + cd ${OSH_PATH:-"../openstack-helm"} && + make all SKIP_CHANGELOG=1 +) diff --git a/tools/deployment/common/ldap.sh b/tools/deployment/common/ldap.sh new file mode 100755 index 0000000000..9bd7c44c03 --- /dev/null +++ b/tools/deployment/common/ldap.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +: ${OSH_INFRA_HELM_REPO:="../openstack-helm-infra"} +: ${OSH_INFRA_VALUES_OVERRIDES_PATH:="../openstack-helm-infra/values_overrides"} +: ${OSH_INFRA_EXTRA_HELM_ARGS_LDAP:="$(helm osh get-values-overrides -p ${OSH_INFRA_VALUES_OVERRIDES_PATH} -c ldap ${FEATURES})"} +: ${NAMESPACE:="osh-infra"} + +#NOTE: Deploy command +helm upgrade --install ldap ${OSH_INFRA_HELM_REPO}/ldap \ + --namespace=${NAMESPACE} \ + --set bootstrap.enabled=true \ + ${OSH_INFRA_EXTRA_HELM_ARGS:=} \ + ${OSH_INFRA_EXTRA_HELM_ARGS_LDAP} + +#NOTE: Wait for deploy +helm osh wait-for-pods ${NAMESPACE} diff --git a/tools/deployment/common/memcached.sh b/tools/deployment/common/memcached.sh new file mode 100755 index 0000000000..d06d5c77a8 --- /dev/null +++ b/tools/deployment/common/memcached.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +#NOTE: Define variables +: ${OSH_INFRA_HELM_REPO:="../openstack-helm-infra"} +: ${OSH_INFRA_VALUES_OVERRIDES_PATH:="../openstack-helm-infra/values_overrides"} +: ${OSH_INFRA_EXTRA_HELM_ARGS_MEMCACHED:="$(helm osh get-values-overrides ${DOWNLOAD_OVERRIDES:-} -p ${OSH_INFRA_VALUES_OVERRIDES_PATH} -c memcached ${FEATURES})"} + +#NOTE: Deploy command +helm upgrade --install memcached ${OSH_INFRA_HELM_REPO}/memcached \ + --namespace=openstack \ + ${OSH_INFRA_EXTRA_HELM_ARGS} \ + ${OSH_INFRA_EXTRA_HELM_ARGS_MEMCACHED} + +#NOTE: Wait for deploy +helm osh wait-for-pods openstack diff --git a/tools/deployment/common/metacontroller.sh b/tools/deployment/common/metacontroller.sh new file mode 100755 index 0000000000..f74e46d5ee --- /dev/null +++ b/tools/deployment/common/metacontroller.sh @@ -0,0 +1,65 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +set -xe + +namespace="metacontroller" +: ${OSH_INFRA_HELM_REPO:="../openstack-helm-infra"} +: ${OSH_INFRA_VALUES_OVERRIDES_PATH:="../openstack-helm-infra/values_overrides"} +: ${HELM_ARGS_METACONTROLLER:="$(helm osh get-values-overrides -p ${OSH_INFRA_VALUES_OVERRIDES_PATH} -c metacontroller ${FEATURES})"} + +#NOTE: Check no crd exists of APIGroup metacontroller.k8s.io +crds=$(kubectl get crd | awk '/metacontroller.k8s.io/{print $1}') + +if [ -z "$crds" ]; then + echo "No crd exists of APIGroup metacontroller.k8s.io" +fi + +tee /tmp/${namespace}-ns.yaml << EOF +apiVersion: v1 +kind: Namespace +metadata: + labels: + kubernetes.io/metadata.name: ${namespace} + name: ${namespace} + name: ${namespace} +EOF + +kubectl create -f /tmp/${namespace}-ns.yaml + +#NOTE: Deploy command +helm upgrade --install metacontroller ${OSH_INFRA_HELM_REPO}/metacontroller \ + --namespace=$namespace \ + --set pod.replicas.metacontroller=3 \ + ${HELM_ARGS_METACONTROLLER} + +#NOTE: Wait for deploy +helm osh wait-for-pods metacontroller + +#NOTE: Check crds of APIGroup metacontroller.k8s.io successfully created +crds=$(kubectl get crd | awk '/metacontroller.k8s.io/{print $1}') + +COUNTER=0 +for i in $crds +do + case $i in + "compositecontrollers.metacontroller.k8s.io") COUNTER=$((COUNTER+1));; + "controllerrevisions.metacontroller.k8s.io") COUNTER=$((COUNTER+1));; + "decoratorcontrollers.metacontroller.k8s.io") COUNTER=$((COUNTER+1));; + *) echo "This is a wrong crd!!!";; + esac +done + +if test $COUNTER -eq 3; then + echo "crds created succesfully" +fi diff --git a/tools/deployment/common/namespace-config.sh b/tools/deployment/common/namespace-config.sh new file mode 100755 index 0000000000..e4d5f77858 --- /dev/null +++ b/tools/deployment/common/namespace-config.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +: ${OSH_INFRA_HELM_REPO:="../openstack-helm-infra"} + +#NOTE: Deploy namespace configs +for NAMESPACE in kube-system ceph openstack; do + helm upgrade --install ${NAMESPACE}-namespace-config ${OSH_INFRA_HELM_REPO}/namespace-config \ + --namespace=${NAMESPACE} +done diff --git a/tools/deployment/common/nfs-provisioner.sh b/tools/deployment/common/nfs-provisioner.sh new file mode 100755 index 0000000000..ad6b3e93cb --- /dev/null +++ b/tools/deployment/common/nfs-provisioner.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +: ${OSH_INFRA_HELM_REPO:="../openstack-helm-infra"} + +tee /tmp/nfs-ns.yaml << EOF +apiVersion: v1 +kind: Namespace +metadata: + labels: + kubernetes.io/metadata.name: nfs + name: nfs + name: nfs +EOF + +kubectl create -f /tmp/nfs-ns.yaml + +#NOTE: Deploy nfs instance for logging, monitoring and alerting components +tee /tmp/nfs-provisioner.yaml << EOF +labels: + node_selector_key: openstack-control-plane + node_selector_value: enabled +storageclass: + name: general +EOF +helm upgrade --install nfs-provisioner \ + ${OSH_INFRA_HELM_REPO}/nfs-provisioner --namespace=nfs \ + --values=/tmp/nfs-provisioner.yaml + +#NOTE: Wait for deployment +helm osh wait-for-pods nfs diff --git a/tools/deployment/common/rabbitmq.sh b/tools/deployment/common/rabbitmq.sh new file mode 100755 index 0000000000..228f69e921 --- /dev/null +++ b/tools/deployment/common/rabbitmq.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +#NOTE: Define variables +: ${OSH_INFRA_HELM_REPO:="../openstack-helm-infra"} +: ${OSH_INFRA_VALUES_OVERRIDES_PATH:="../openstack-helm-infra/values_overrides"} +: ${OSH_INFRA_EXTRA_HELM_ARGS_RABBITMQ:="$(helm osh get-values-overrides ${DOWNLOAD_OVERRIDES:-} -p ${OSH_INFRA_VALUES_OVERRIDES_PATH} -c rabbitmq ${FEATURES})"} +: ${NAMESPACE:=openstack} + +#NOTE: Deploy command +helm upgrade --install rabbitmq ${OSH_INFRA_HELM_REPO}/rabbitmq \ + --namespace=${NAMESPACE} \ + --set pod.replicas.server=1 \ + --timeout=600s \ + ${VOLUME_HELM_ARGS:="--set volume.enabled=false --set volume.use_local_path.enabled=true"} \ + ${OSH_INFRA_EXTRA_HELM_ARGS:=} \ + ${OSH_INFRA_EXTRA_HELM_ARGS_RABBITMQ} + +#NOTE: Wait for deploy +helm osh wait-for-pods ${NAMESPACE} + +helm test rabbitmq --namespace openstack diff --git a/tools/deployment/common/sleep.sh b/tools/deployment/common/sleep.sh new file mode 100755 index 0000000000..cb8fe16b49 --- /dev/null +++ b/tools/deployment/common/sleep.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +set -ex + +while true; do + echo "Sleeping for 100 seconds..." +done diff --git a/tools/deployment/db/mariadb-backup.sh b/tools/deployment/db/mariadb-backup.sh new file mode 100755 index 0000000000..a812eb0ade --- /dev/null +++ b/tools/deployment/db/mariadb-backup.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +: ${OSH_INFRA_HELM_REPO:="../openstack-helm-infra"} +: ${OSH_INFRA_VALUES_OVERRIDES_PATH:="../openstack-helm-infra/values_overrides"} +: ${OSH_INFRA_EXTRA_HELM_ARGS_MARIADB_BACKUP:="$(helm osh get-values-overrides -c mariadb-backup ${FEATURES})"} + +#NOTE: Deploy command +helm upgrade --install mariadb-backup ${OSH_INFRA_HELM_REPO}/mariadb-backup \ + --namespace=openstack \ + --wait \ + --timeout 900s \ + ${OSH_INFRA_EXTRA_HELM_ARGS:=} \ + ${OSH_INFRA_EXTRA_HELM_ARGS_MARIADB_BACKUP} + +helm osh wait-for-pods openstack + +kubectl create job --from=cronjob/mariadb-backup mariadb-backup-manual-001 -n openstack + +helm osh wait-for-pods openstack + +kubectl logs jobs/mariadb-backup-manual-001 -n openstack diff --git a/tools/deployment/db/mariadb-operator-cluster.sh b/tools/deployment/db/mariadb-operator-cluster.sh new file mode 100755 index 0000000000..da17daffaa --- /dev/null +++ b/tools/deployment/db/mariadb-operator-cluster.sh @@ -0,0 +1,70 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +# Specify the Rook release tag to use for the Rook operator here +: ${MARIADB_OPERATOR_RELEASE:="0.22.0"} + +# install mariadb-operator +helm repo add mariadb-operator https://mariadb-operator.github.io/mariadb-operator +helm upgrade --install --create-namespace mariadb-operator mariadb-operator/mariadb-operator --version ${MARIADB_OPERATOR_RELEASE} -n mariadb-operator + +#NOTE: Wait for deploy +helm osh wait-for-pods mariadb-operator + +: ${OSH_INFRA_HELM_REPO:="../openstack-helm-infra"} +: ${OSH_INFRA_VALUES_OVERRIDES_PATH:="../openstack-helm-infra/values_overrides"} +: ${OSH_INFRA_EXTRA_HELM_ARGS_MARIADB_CLUSTER:="$(helm osh get-values-overrides -p ${OSH_INFRA_VALUES_OVERRIDES_PATH} -c mariadb-cluster ${FEATURES})"} + +#NOTE: Deploy command +# Deploying downscaled cluster +: ${OSH_INFRA_EXTRA_HELM_ARGS:=""} +helm upgrade --install mariadb-cluster ${OSH_INFRA_HELM_REPO}/mariadb-cluster \ + --namespace=openstack \ + --wait \ + --timeout 900s \ + --values values_overrides/mariadb-cluster/downscaled.yaml \ + ${OSH_INFRA_EXTRA_HELM_ARGS} \ + ${OSH_INFRA_EXTRA_HELM_ARGS_MARIADB_CLUSTER} + +sleep 30 + +#NOTE: Wait for deploy +helm osh wait-for-pods openstack + +kubectl get pods --namespace=openstack -o wide + +#NOTE: Deploy command +# Upscaling the cluster to 3 instances +# mariadb-operator is not handinling changes in appropriate statefulset +# so a special job has to delete the statefulset in order +# to let mariadb-operator to re-create the sts with new params +helm upgrade --install mariadb-cluster ./mariadb-cluster \ + --namespace=openstack \ + --wait \ + --timeout 900s \ + --values values_overrides/mariadb-cluster/upscaled.yaml \ + ${OSH_INFRA_EXTRA_HELM_ARGS} \ + ${OSH_INFRA_EXTRA_HELM_ARGS_MARIADB_CLUSTER} + +#NOTE: Wait for deploy +helm osh wait-for-pods openstack + +kubectl get pods --namespace=openstack -o wide + +# Delete the test pod if it still exists +kubectl delete pods -l application=mariadb,release_group=mariadb-cluster,component=test --namespace=openstack --ignore-not-found +#NOTE: Validate the deployment +helm test mariadb-cluster --namespace openstack diff --git a/tools/deployment/db/mariadb.sh b/tools/deployment/db/mariadb.sh new file mode 100755 index 0000000000..f23737f6cd --- /dev/null +++ b/tools/deployment/db/mariadb.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +: ${OSH_INFRA_HELM_REPO:="../openstack-helm-infra"} +: ${OSH_INFRA_VALUES_OVERRIDES_PATH:="../openstack-helm-infra/values_overrides"} +: ${OSH_INFRA_EXTRA_HELM_ARGS_MARIADB:="$(helm osh get-values-overrides ${DOWNLOAD_OVERRIDES:-} -p ${OSH_INFRA_VALUES_OVERRIDES_PATH} -c mariadb ${FEATURES})"} +: ${NAMESPACE:="osh-infra"} +: ${RUN_HELM_TESTS:="yes"} + +#NOTE: Deploy command +helm upgrade --install mariadb ${OSH_INFRA_HELM_REPO}/mariadb \ + --namespace=${NAMESPACE} \ + ${MONITORING_HELM_ARGS:="--set monitoring.prometheus.enabled=true"} \ + --timeout=600s \ + ${OSH_INFRA_EXTRA_HELM_ARGS:=} \ + ${OSH_INFRA_EXTRA_HELM_ARGS_MARIADB} + +#NOTE: Wait for deploy +helm osh wait-for-pods ${NAMESPACE} + +if [ "x${RUN_HELM_TESTS}" != "xno" ]; then + # Delete the test pod if it still exists + kubectl delete pods -l application=mariadb,release_group=mariadb,component=test --namespace=${NAMESPACE} --ignore-not-found + #NOTE: Validate the deployment + helm test mariadb --namespace ${NAMESPACE} +fi diff --git a/tools/deployment/db/postgresql.sh b/tools/deployment/db/postgresql.sh new file mode 100755 index 0000000000..80fccd9567 --- /dev/null +++ b/tools/deployment/db/postgresql.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +#NOTE: Deploy command +: ${OSH_INFRA_HELM_REPO:="../openstack-helm-infra"} +: ${OSH_INFRA_VALUES_OVERRIDES_PATH:="../openstack-helm-infra/values_overrides"} +: ${OSH_INFRA_EXTRA_HELM_ARGS:=""} +: ${OSH_INFRA_EXTRA_HELM_ARGS_POSTGRESQL:="$(helm osh get-values-overrides -p ${OSH_INFRA_VALUES_OVERRIDES_PATH} -c postgresql ${FEATURES})"} + +helm upgrade --install postgresql ${OSH_INFRA_HELM_REPO}/postgresql \ + --namespace=osh-infra \ + --set monitoring.prometheus.enabled=true \ + --set storage.pvc.size=1Gi \ + --set storage.pvc.enabled=true \ + --set pod.replicas.server=1 \ + ${OSH_INFRA_EXTRA_HELM_ARGS} \ + ${OSH_INFRA_EXTRA_HELM_ARGS_POSTGRESQL} + +#NOTE: Wait for deploy +helm osh wait-for-pods osh-infra diff --git a/tools/deployment/logging/elasticsearch.sh b/tools/deployment/logging/elasticsearch.sh new file mode 100755 index 0000000000..5e3b19bee0 --- /dev/null +++ b/tools/deployment/logging/elasticsearch.sh @@ -0,0 +1,191 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +#NOTE: Deploy command +tee /tmp/elasticsearch.yaml << EOF +jobs: + verify_repositories: + cron: "*/3 * * * *" +monitoring: + prometheus: + enabled: true +pod: + replicas: + client: 1 + data: 1 + master: 2 +conf: + elasticsearch: + snapshots: + enabled: true + api_objects: + snapshot_repo: + endpoint: _snapshot/ceph-rgw + body: + type: s3 + settings: + client: default + bucket: elasticsearch-bucket + slm_policy: + endpoint: _slm/policy/snapshots + body: + schedule: "0 */15 * * * ?" + name: "" + repository: ceph-rgw + config: + indices: + - "<*-{now/d}>" + retention: + expire_after: 30d + ilm_policy: + endpoint: _ilm/policy/cleanup + body: + policy: + phases: + delete: + min_age: 5d + actions: + delete: {} + test_empty: {} +storage: + s3: + clients: + # These values configure the s3 clients section of elasticsearch.yml, with access_key and secret_key being saved to the keystore + default: + # not needed when using Rook Ceph CRDs + # auth: + # username: elasticsearch + # access_key: "elastic_access_key" + # secret_key: "elastic_secret_key" + settings: + # endpoint: Defaults to the ceph-rgw endpoint + # protocol: Defaults to http + path_style_access: true # Required for ceph-rgw S3 API + create_user: true # Attempt to create the user at the ceph_object_store endpoint, authenticating using the secret named at .Values.secrets.rgw.admin + backup: # Change this as you'd like + # not needed when using Rook Ceph CRDs + # auth: + # username: backup + # access_key: "backup_access_key" + # secret_key: "backup_secret_key" + settings: + # endpoint: rook-ceph-rgw-default.ceph.svc.cluster.local # Using the ingress here to test the endpoint override + path_style_access: true + create_user: true + buckets: # List of buckets to create (if required). + - name: elasticsearch-bucket + client: default + storage_class: ceph-bucket # this is valid when using Rook CRDs + # not needed when using Rook Ceph CRDs + # options: # list of extra options for s3cmd + # - --region="default:osh-infra" + - name: backup-bucket + client: backup + storage_class: ceph-bucket # this is valid when using Rook CRDs + # not needed when using Rook Ceph CRDs + # options: # list of extra options for s3cmd + # - --region="default:backup" +endpoints: + ceph_object_store: + name: radosgw + namespace: ceph + hosts: + default: rook-ceph-rgw-default + public: radosgw + host_fqdn_override: + default: null + path: + default: null + scheme: + default: http + port: + api: + default: 8080 + public: 80 +network: + elasticsearch: + ingress: + classes: + namespace: nginx-osh-infra +dependencies: + static: + elasticsearch_templates: + services: + - endpoint: internal + service: elasticsearch + jobs: null + custom_resources: + - apiVersion: objectbucket.io/v1alpha1 + kind: ObjectBucket + name: obc-osh-infra-elasticsearch-bucket + fields: + - key: "status.phase" + value: "Bound" + - apiVersion: objectbucket.io/v1alpha1 + kind: ObjectBucket + name: obc-osh-infra-backup-bucket + fields: + - key: "status.phase" + value: "Bound" + snapshot_repository: + services: + - endpoint: internal + service: elasticsearch + jobs: null + custom_resources: + - apiVersion: objectbucket.io/v1alpha1 + kind: ObjectBucket + name: obc-osh-infra-elasticsearch-bucket + fields: + - key: "status.phase" + value: "Bound" + - apiVersion: objectbucket.io/v1alpha1 + kind: ObjectBucket + name: obc-osh-infra-backup-bucket + fields: + - key: "status.phase" + value: "Bound" +manifests: + job_s3_user: false + job_s3_bucket: false + object_bucket_claim: true + +# FIXME: The kubernetes-entrypoint image used by default +# quay.io/airshipit/kubernetes-entrypoint:latest-ubuntu_focal +# can not lookup for global (w/o namespace) custom resources +# but ObjectBucket CRs are global and we have them as dependencies +# for two elasticsearch jobs. +images: + tags: + dep_check: quay.io/airshipit/kubernetes-entrypoint:v1.0.0 +EOF + +: ${OSH_INFRA_HELM_REPO:="../openstack-helm-infra"} +: ${OSH_INFRA_VALUES_OVERRIDES_PATH:="../openstack-helm-infra/values_overrides"} +: ${OSH_INFRA_EXTRA_HELM_ARGS_ELASTICSEARCH:="$(helm osh get-values-overrides -p ${OSH_INFRA_VALUES_OVERRIDES_PATH} -c elasticsearch ${FEATURES})"} + +helm upgrade --install elasticsearch ${OSH_INFRA_HELM_REPO}/elasticsearch \ + --namespace=osh-infra \ + --values=/tmp/elasticsearch.yaml\ + ${OSH_INFRA_EXTRA_HELM_ARGS} \ + ${OSH_INFRA_EXTRA_HELM_ARGS_ELASTICSEARCH} + +#NOTE: Wait for deploy +helm osh wait-for-pods osh-infra + +# Delete the test pod if it still exists +kubectl delete pods -l application=elasticsearch,release_group=elasticsearch,component=test --namespace=osh-infra --ignore-not-found +helm test elasticsearch --namespace osh-infra diff --git a/tools/deployment/logging/fluentbit.sh b/tools/deployment/logging/fluentbit.sh new file mode 100755 index 0000000000..84692850d8 --- /dev/null +++ b/tools/deployment/logging/fluentbit.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +: ${OSH_INFRA_HELM_REPO:="../openstack-helm-infra"} +: ${OSH_INFRA_VALUES_OVERRIDES_PATH:="../openstack-helm-infra/values_overrides"} +: ${OSH_INFRA_EXTRA_HELM_ARGS_FLUENTBIT:="$(helm osh get-values-overrides -p ${OSH_INFRA_VALUES_OVERRIDES_PATH} -c fluentbit ${FEATURES})"} + +helm upgrade --install fluentbit ${OSH_INFRA_HELM_REPO}/fluentbit \ + --namespace=osh-infra \ + ${OSH_INFRA_EXTRA_HELM_ARGS:=} \ + ${OSH_INFRA_EXTRA_HELM_ARGS_FLUENTBIT} + +#NOTE: Wait for deploy +helm osh wait-for-pods osh-infra diff --git a/tools/deployment/logging/fluentd.sh b/tools/deployment/logging/fluentd.sh new file mode 100755 index 0000000000..2002013af4 --- /dev/null +++ b/tools/deployment/logging/fluentd.sh @@ -0,0 +1,236 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +: ${OSH_INFRA_HELM_REPO:="../openstack-helm-infra"} +: ${OSH_INFRA_VALUES_OVERRIDES_PATH:="../openstack-helm-infra/values_overrides"} +: ${OSH_INFRA_EXTRA_HELM_ARGS_FLUENTD:="$(helm osh get-values-overrides -p ${OSH_INFRA_VALUES_OVERRIDES_PATH} -c fluentd ${FEATURES})"} + +tee /tmp/fluentd.yaml << EOF +pod: + env: + fluentd: + vars: + MY_TEST_VAR: FOO + secrets: + MY_TEST_SECRET: BAR +conf: + fluentd: + conf: + # These fields are rendered as helm templates + input: | + + @type prometheus + port {{ tuple "fluentd" "internal" "metrics" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + + + + @type prometheus_monitor + + + + @type prometheus_output_monitor + + + + @type prometheus_tail_monitor + + + + bind 0.0.0.0 + port "#{ENV['FLUENTD_PORT']}" + @type forward + + + + @type tail + @id in_tail_container_logs + path "/var/log/containers/*.log" + pos_file "/var/log/fluentd-containers.log.pos" + tag kubernetes.* + read_from_head true + emit_unmatched_lines true + + @type "multi_format" + + format json + time_key "time" + time_type string + time_format "%Y-%m-%dT%H:%M:%S.%NZ" + keep_time_key false + + + format regexp + expression /^(? + + + + + @type tail + tag libvirt.* + path /var/log/libvirt/**.log + pos_file "/var/log/fluentd-libvirt.log.pos" + read_from_head true + + @type none + + + + + @type systemd + tag auth + path /var/log/journal + matches [{ "SYSLOG_FACILITY":"10" }] + read_from_head true + + + @type local + path /var/log/fluentd-systemd-auth.json + + + + fields_strip_underscores true + fields_lowercase true + + + + + @type systemd + tag journal.* + path /var/log/journal + matches [{ "_SYSTEMD_UNIT": "docker.service" }] + read_from_head true + + + @type local + path /var/log/fluentd-systemd-docker.json + + + + fields_strip_underscores true + fields_lowercase true + + + + + @type systemd + tag journal.* + path /var/log/journal + matches [{ "_SYSTEMD_UNIT": "kubelet.service" }] + read_from_head true + + + @type local + path /var/log/fluentd-systemd-kubelet.json + + + + fields_strip_underscores true + fields_lowercase true + + + + + @type systemd + tag kernel + path /var/log/journal + matches [{ "_TRANSPORT": "kernel" }] + read_from_head true + + + @type local + path /var/log/fluentd-systemd-kernel.json + + + + fields_strip_underscores true + fields_lowercase true + + + + + @type relabel + @label @filter + + + filter: | + + + + output: | + +EOF +helm upgrade --install fluentd ${OSH_INFRA_HELM_REPO}/fluentd \ + --namespace=osh-infra \ + --values=/tmp/fluentd.yaml \ + ${OSH_INFRA_EXTRA_HELM_ARGS} \ + ${OSH_INFRA_EXTRA_HELM_ARGS_FLUENTD} + +#NOTE: Wait for deploy +helm osh wait-for-pods osh-infra diff --git a/tools/deployment/logging/kibana.sh b/tools/deployment/logging/kibana.sh new file mode 100755 index 0000000000..dfbe90c32a --- /dev/null +++ b/tools/deployment/logging/kibana.sh @@ -0,0 +1,29 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +: ${OSH_INFRA_HELM_REPO:="../openstack-helm-infra"} +: ${OSH_INFRA_VALUES_OVERRIDES_PATH:="../openstack-helm-infra/values_overrides"} +: ${OSH_INFRA_EXTRA_HELM_ARGS_KIBANA:="$(helm osh get-values-overrides -p ${OSH_INFRA_VALUES_OVERRIDES_PATH} -c kibana ${FEATURES})"} + +#NOTE: Deploy command +helm upgrade --install kibana ${OSH_INFRA_HELM_REPO}/kibana \ + --namespace=osh-infra \ + --set network.kibana.ingress.classes.namespace=nginx-osh-infra \ + ${OSH_INFRA_EXTRA_HELM_ARGS} \ + ${OSH_INFRA_EXTRA_HELM_ARGS_KIBANA} + +#NOTE: Wait for deploy +helm osh wait-for-pods osh-infra diff --git a/tools/deployment/monitoring/alertmanager.sh b/tools/deployment/monitoring/alertmanager.sh new file mode 100755 index 0000000000..e2cbc8db56 --- /dev/null +++ b/tools/deployment/monitoring/alertmanager.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +: ${OSH_INFRA_HELM_REPO:="../openstack-helm-infra"} + +#NOTE: Deploy command +helm upgrade --install prometheus-alertmanager ${OSH_INFRA_HELM_REPO}/prometheus-alertmanager \ + --namespace=osh-infra \ + --set pod.replicas.alertmanager=1 + +#NOTE: Wait for deploy +helm osh wait-for-pods osh-infra diff --git a/tools/deployment/monitoring/blackbox-exporter.sh b/tools/deployment/monitoring/blackbox-exporter.sh new file mode 100755 index 0000000000..85fb496ebf --- /dev/null +++ b/tools/deployment/monitoring/blackbox-exporter.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +: ${OSH_INFRA_HELM_REPO:="../openstack-helm-infra"} + +#NOTE: Deploy command +helm upgrade --install prometheus-blackbox-exporter \ + ${OSH_INFRA_HELM_REPO}/prometheus-blackbox-exporter --namespace=osh-infra + +#NOTE: Wait for deploy +helm osh wait-for-pods osh-infra diff --git a/tools/deployment/monitoring/grafana.sh b/tools/deployment/monitoring/grafana.sh new file mode 100755 index 0000000000..73f6e2da2b --- /dev/null +++ b/tools/deployment/monitoring/grafana.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +: ${OSH_INFRA_HELM_REPO:="../openstack-helm-infra"} +: ${OSH_INFRA_VALUES_OVERRIDES_PATH:="../openstack-helm-infra/values_overrides"} +FEATURES="calico ceph containers coredns elasticsearch kubernetes nginx nodes openstack prometheus home_dashboard persistentvolume apparmor ${FEATURES}" +: ${OSH_INFRA_EXTRA_HELM_ARGS_GRAFANA:=$(helm osh get-values-overrides -p ${OSH_INFRA_VALUES_OVERRIDES_PATH} -c grafana ${FEATURES} 2>/dev/null)} + +#NOTE: Deploy command +helm upgrade --install grafana ${OSH_INFRA_HELM_REPO}/grafana \ + --namespace=osh-infra \ + ${OSH_INFRA_EXTRA_HELM_ARGS:=} \ + ${OSH_INFRA_EXTRA_HELM_ARGS_GRAFANA} + +#NOTE: Wait for deploy +helm osh wait-for-pods osh-infra + +# Delete the test pod if it still exists +kubectl delete pods -l application=grafana,release_group=grafana,component=test --namespace=osh-infra --ignore-not-found +helm test grafana --namespace osh-infra diff --git a/tools/deployment/monitoring/kube-state-metrics.sh b/tools/deployment/monitoring/kube-state-metrics.sh new file mode 100755 index 0000000000..c917877300 --- /dev/null +++ b/tools/deployment/monitoring/kube-state-metrics.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +: ${OSH_INFRA_HELM_REPO:="../openstack-helm-infra"} +: ${OSH_INFRA_VALUES_OVERRIDES_PATH:="../openstack-helm-infra/values_overrides"} +: ${OSH_INFRA_EXTRA_HELM_ARGS_KUBE_STATE_METRICS:="$(helm osh get-values-overrides -p ${OSH_INFRA_VALUES_OVERRIDES_PATH} -c prometheus-kube-state-metrics ${FEATURES})"} + +#NOTE: Deploy command +helm upgrade --install prometheus-kube-state-metrics \ + ${OSH_INFRA_HELM_REPO}/prometheus-kube-state-metrics --namespace=kube-system \ + ${OSH_INFRA_EXTRA_HELM_ARGS_KUBE_STATE_METRICS} + +#NOTE: Wait for deploy +helm osh wait-for-pods kube-system diff --git a/tools/deployment/monitoring/mysql-exporter.sh b/tools/deployment/monitoring/mysql-exporter.sh new file mode 100755 index 0000000000..280d76bf8f --- /dev/null +++ b/tools/deployment/monitoring/mysql-exporter.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +: ${OSH_INFRA_HELM_REPO:="../openstack-helm-infra"} +: ${OSH_INFRA_VALUES_OVERRIDES_PATH:="../openstack-helm-infra/values_overrides"} +: ${OSH_INFRA_EXTRA_HELM_ARGS_MARIADB_MYSQL_EXPORTER:="$(helm osh get-values-overrides -p ${OSH_INFRA_VALUES_OVERRIDES_PATH} -c prometheus-mysql-exporter ${FEATURES})"} + +#NOTE: Deploy command +helm upgrade --install prometheus-mysql-exporter ${OSH_INFRA_HELM_REPO}/prometheus-mysql-exporter \ + --namespace=openstack \ + --wait \ + --timeout 900s \ + ${OSH_INFRA_EXTRA_HELM_ARGS:=} \ + ${OSH_INFRA_EXTRA_HELM_ARGS_MARIADB_MYSQL_EXPORTER} + + +#NOTE: Wait for deploy +helm osh wait-for-pods openstack + +kubectl get pods --namespace=openstack -o wide diff --git a/tools/deployment/monitoring/nagios.sh b/tools/deployment/monitoring/nagios.sh new file mode 100755 index 0000000000..c43f8cfc8c --- /dev/null +++ b/tools/deployment/monitoring/nagios.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +: ${OSH_INFRA_HELM_REPO:="../openstack-helm-infra"} +: ${OSH_INFRA_VALUES_OVERRIDES_PATH:="../openstack-helm-infra/values_overrides"} +: ${OSH_INFRA_EXTRA_HELM_ARGS_NAGIOS:="$(helm osh get-values-overrides -p ${OSH_INFRA_VALUES_OVERRIDES_PATH} -c nagios ${FEATURES})"} + +#NOTE: Deploy command +helm upgrade --install nagios ${OSH_INFRA_HELM_REPO}/nagios \ + --namespace=osh-infra \ + ${OSH_INFRA_EXTRA_HELM_ARGS:=} \ + ${OSH_INFRA_EXTRA_HELM_ARGS_NAGIOS} + +#NOTE: Wait for deploy +helm osh wait-for-pods osh-infra + +# Delete the test pod if it still exists +kubectl delete pods -l application=nagios,release_group=nagios,component=test --namespace=osh-infra --ignore-not-found +helm test nagios --namespace osh-infra diff --git a/tools/deployment/monitoring/node-exporter.sh b/tools/deployment/monitoring/node-exporter.sh new file mode 100755 index 0000000000..704605325d --- /dev/null +++ b/tools/deployment/monitoring/node-exporter.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +: ${OSH_INFRA_HELM_REPO:="../openstack-helm-infra"} +: ${OSH_INFRA_VALUES_OVERRIDES_PATH:="../openstack-helm-infra/values_overrides"} +: ${OSH_INFRA_EXTRA_HELM_ARGS_NODE_EXPORTER:="$(helm osh get-values-overrides -p ${OSH_INFRA_VALUES_OVERRIDES_PATH} -c prometheus-node-exporter ${FEATURES})"} + +#NOTE: Deploy command +helm upgrade --install prometheus-node-exporter \ + ${OSH_INFRA_HELM_REPO}/prometheus-node-exporter --namespace=kube-system \ + ${OSH_INFRA_EXTRA_HELM_ARGS_NODE_EXPORTER} + +#NOTE: Wait for deploy +helm osh wait-for-pods kube-system diff --git a/tools/deployment/monitoring/node-problem-detector.sh b/tools/deployment/monitoring/node-problem-detector.sh new file mode 100755 index 0000000000..6dc08a4f3f --- /dev/null +++ b/tools/deployment/monitoring/node-problem-detector.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +: ${OSH_INFRA_HELM_REPO:="../openstack-helm-infra"} + +#NOTE: Deploy command +tee /tmp/kubernetes-node-problem-detector.yaml << EOF +monitoring: + prometheus: + pod: + enabled: false + service: + enabled: true +manifests: + service: true +EOF +helm upgrade --install kubernetes-node-problem-detector \ + ${OSH_INFRA_HELM_REPO}/kubernetes-node-problem-detector --namespace=kube-system \ + --values=/tmp/kubernetes-node-problem-detector.yaml + +#NOTE: Wait for deploy +helm osh wait-for-pods kube-system diff --git a/tools/deployment/monitoring/openstack-exporter.sh b/tools/deployment/monitoring/openstack-exporter.sh new file mode 100755 index 0000000000..004348b457 --- /dev/null +++ b/tools/deployment/monitoring/openstack-exporter.sh @@ -0,0 +1,39 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +: ${OSH_INFRA_HELM_REPO:="../openstack-helm-infra"} +: ${OSH_INFRA_VALUES_OVERRIDES_PATH:="../openstack-helm-infra/values_overrides"} +: ${OSH_INFRA_EXTRA_HELM_ARGS_OS_EXPORTER:="$(helm osh get-values-overrides -p ${OSH_INFRA_VALUES_OVERRIDES_PATH} -c prometheus-openstack-exporter ${FEATURES})"} + +tee /tmp/prometheus-openstack-exporter.yaml << EOF +manifests: + job_ks_user: false +dependencies: + static: + prometheus_openstack_exporter: + jobs: null + services: null +EOF + +#NOTE: Deploy command +helm upgrade --install prometheus-openstack-exporter \ + ${OSH_INFRA_HELM_REPO}/prometheus-openstack-exporter \ + --namespace=openstack \ + --values=/tmp/prometheus-openstack-exporter.yaml \ + ${OSH_INFRA_EXTRA_HELM_ARGS_OS_EXPORTER} + +#NOTE: Wait for deploy +helm osh wait-for-pods openstack diff --git a/tools/deployment/monitoring/process-exporter.sh b/tools/deployment/monitoring/process-exporter.sh new file mode 100755 index 0000000000..1f27271723 --- /dev/null +++ b/tools/deployment/monitoring/process-exporter.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +: ${OSH_INFRA_HELM_REPO:="../openstack-helm-infra"} +: ${OSH_INFRA_VALUES_OVERRIDES_PATH:="../openstack-helm-infra/values_overrides"} +: ${OSH_INFRA_EXTRA_HELM_ARGS_PROCESS_EXPORTER:="$(helm osh get-values-overrides -p ${OSH_INFRA_VALUES_OVERRIDES_PATH} -c prometheus-process-exporter ${FEATURES})"} + +#NOTE: Deploy command +helm upgrade --install prometheus-process-exporter \ + ${OSH_INFRA_HELM_REPO}/prometheus-process-exporter --namespace=kube-system \ + ${OSH_INFRA_EXTRA_HELM_ARGS_PROCESS_EXPORTER} + +#NOTE: Wait for deploy +helm osh wait-for-pods kube-system diff --git a/tools/deployment/monitoring/prometheus.sh b/tools/deployment/monitoring/prometheus.sh new file mode 100755 index 0000000000..00fa49a140 --- /dev/null +++ b/tools/deployment/monitoring/prometheus.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +: ${OSH_INFRA_HELM_REPO:="../openstack-helm-infra"} +: ${OSH_INFRA_VALUES_OVERRIDES_PATH:="../openstack-helm-infra/values_overrides"} +FEATURES="alertmanager ceph elasticsearch kubernetes nodes openstack postgresql apparmor ${FEATURES}" +: ${OSH_INFRA_EXTRA_HELM_ARGS_PROMETHEUS:="$(helm osh get-values-overrides -p ${OSH_INFRA_VALUES_OVERRIDES_PATH} -c prometheus ${FEATURES})"} + +#NOTE: Deploy command +helm upgrade --install prometheus ${OSH_INFRA_HELM_REPO}/prometheus \ + --namespace=osh-infra \ + ${OSH_INFRA_EXTRA_HELM_ARGS:=} \ + ${OSH_INFRA_EXTRA_HELM_ARGS_PROMETHEUS} + +#NOTE: Wait for deploy +helm osh wait-for-pods osh-infra + +# Delete the test pod if it still exists +kubectl delete pods -l application=prometheus,release_group=prometheus,component=test --namespace=osh-infra --ignore-not-found +helm test prometheus --namespace osh-infra diff --git a/tools/deployment/openstack/keystone.sh b/tools/deployment/openstack/keystone.sh new file mode 100755 index 0000000000..f285f90a65 --- /dev/null +++ b/tools/deployment/openstack/keystone.sh @@ -0,0 +1,32 @@ +#!/bin/bash + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +set -xe + +: ${OSH_HELM_REPO:="../openstack-helm"} +: ${OSH_VALUES_OVERRIDES_PATH:="../openstack-helm/values_overrides"} +: ${OSH_EXTRA_HELM_ARGS_KEYSTONE:="$(helm osh get-values-overrides ${DOWNLOAD_OVERRIDES:-} -p ${OSH_VALUES_OVERRIDES_PATH} -c keystone ${FEATURES})"} + +# Install Keystone +helm upgrade --install keystone ${OSH_HELM_REPO}/keystone \ + --namespace=openstack \ + ${OSH_EXTRA_HELM_ARGS:=} \ + ${OSH_EXTRA_HELM_ARGS_KEYSTONE} + +helm osh wait-for-pods openstack + +# Testing basic functionality +export OS_CLOUD=openstack_helm +sleep 30 #NOTE(portdirect): Wait for ingress controller to update rules and restart Nginx +openstack endpoint list diff --git a/tools/gate/lint.sh b/tools/gate/lint.sh new file mode 100755 index 0000000000..8e7e4ce6fa --- /dev/null +++ b/tools/gate/lint.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +set -e + +HELM_DATA_YAML=../openstack-helm-infra/roles/build-helm-packages/defaults/main.yml +HELM_VERSION=$(yq -r '.version.helm' ${HELM_DATA_YAML}) +HELM_REPO_URL=$(yq -r '.url.helm_repo' ${HELM_DATA_YAML}) +LINT_DIR=.yamllint + +rm -rf */charts/helm-toolkit +mkdir ${LINT_DIR} +cp -r * ${LINT_DIR} +rm -rf ${LINT_DIR}/*/templates +wget -qO ${LINT_DIR}/helm.tgz ${HELM_REPO_URL}/helm-${HELM_VERSION}-linux-amd64.tar.gz +tar xzf ${LINT_DIR}/helm.tgz -C ${LINT_DIR} --strip-components=1 linux-amd64/helm + +for i in */; do + # avoid helm-toolkit to symlink on itself + [ -d "$i/templates" -a "$i" != "helm-toolkit/" ] || continue + mkdir -p $i/charts + ln -s ../../../openstack-helm-infra/helm-toolkit $i/charts/helm-toolkit + ${LINT_DIR}/helm template $i --output-dir ${LINT_DIR} 2>&1 > /dev/null +done +rm -rf */charts/helm-toolkit + +find .yamllint -type f -exec sed -i 's/%%%.*/XXX/g' {} + + +set +e +shopt -s globstar extglob +# lint all y*mls except for templates with the first config +yamllint -c yamllint.conf ${LINT_DIR}/*{,/!(templates)/**}/*.y*ml yamllint*.conf +result=$? +# lint templates with the second config +yamllint -c yamllint-templates.conf ${LINT_DIR}/*/templates/*.yaml +exit $(($?|$result)) diff --git a/tools/gate/reno-check.sh b/tools/gate/reno-check.sh new file mode 100755 index 0000000000..cbfdfce931 --- /dev/null +++ b/tools/gate/reno-check.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -e +RESULT=0 +IFS=$'\n' +for chart in $(find $(pwd) -maxdepth 2 -name 'Chart.yaml');do + SERVICE=$(egrep "^name:" "$chart"|awk -F ' ' '{print $2}') + VERSION=$(egrep "^version:" "$chart"|awk -F ' ' '{print $2}') + if grep -q "$VERSION" ./releasenotes/notes/$SERVICE.yaml ; then + echo "$SERVICE is up to date!" + else + echo "$SERVICE version does not match release notes. Likely requires a release note update" + RESULT=1 + fi +done + +exit $RESULT diff --git a/tools/gate/selenium/grafana-selenium.sh b/tools/gate/selenium/grafana-selenium.sh new file mode 100755 index 0000000000..f030c4c95e --- /dev/null +++ b/tools/gate/selenium/grafana-selenium.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +set -xe + +export CHROMEDRIVER="${CHROMEDRIVER:="/etc/selenium/chromedriver"}" +export ARTIFACTS_DIR="${ARTIFACTS_DIR:="/tmp/artifacts/"}" + +export GRAFANA_USER="admin" +export GRAFANA_PASSWORD="password" +export GRAFANA_URI="grafana.osh-infra.svc.cluster.local" + +python3 $(readlink -f $(dirname $0))/grafanaSelenium.py diff --git a/tools/gate/selenium/grafanaSelenium.py b/tools/gate/selenium/grafanaSelenium.py new file mode 100755 index 0000000000..40a5eaa8b2 --- /dev/null +++ b/tools/gate/selenium/grafanaSelenium.py @@ -0,0 +1,51 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +from selenium.webdriver.common.by import By +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support import expected_conditions as EC +from selenium.common.exceptions import TimeoutException +from selenium.common.exceptions import NoSuchElementException +from seleniumtester import SeleniumTester + +st = SeleniumTester('Grafana') + +username = st.get_variable('GRAFANA_USER') +password = st.get_variable('GRAFANA_PASSWORD') +grafana_uri = st.get_variable('GRAFANA_URI') +grafana_url = 'http://{0}'.format(grafana_uri) + +try: + st.logger.info('Attempting to connect to Grafana') + st.browser.get(grafana_url) + el = WebDriverWait(st.browser, 15).until( + EC.title_contains('Grafana') + ) + st.logger.info('Connected to Grafana') +except TimeoutException: + st.logger.critical('Timed out waiting to connect to Grafana') + st.browser.quit() + sys.exit(1) + +st.logger.info("Attempting to log into Grafana dashboard") +try: + st.browser.find_element(By.NAME, 'user').send_keys(username) + st.browser.find_element(By.NAME, 'password').send_keys(password) + st.browser.find_element(By.CLASS_NAME, 'css-1mhnkuh').click() + st.logger.info("Successfully logged in to Grafana") +except NoSuchElementException: + st.logger.error("Failed to log in to Grafana") + st.browser.quit() + sys.exit(1) + +st.browser.quit() diff --git a/tools/gate/selenium/kibana-selenium.sh b/tools/gate/selenium/kibana-selenium.sh new file mode 100755 index 0000000000..37c2bdd757 --- /dev/null +++ b/tools/gate/selenium/kibana-selenium.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -xe + +export CHROMEDRIVER="${CHROMEDRIVER:="/etc/selenium/chromedriver"}" +export ARTIFACTS_DIR="${ARTIFACTS_DIR:="/tmp/artifacts/"}" + +export KIBANA_USER="admin" +export KIBANA_PASSWORD="changeme" +export KIBANA_URI="kibana.osh-infra.svc.cluster.local" + +export KERNEL_QUERY="discove?r_g=()&_a=(columns:!(_source),index:'kernel*',interval:auto,query:(language:kuery,query:''),sort:!('@timestamp',desc))" +export JOURNAL_QUERY="discove?r_g=()&_a=(columns:!(_source),index:'journal*',interval:auto,query:(language:kuery,query:''),sort:!('@timestamp',desc))" +export LOGSTASH_QUERY="discove?r_g=()&_a=(columns:!(_source),index:'logstash*',interval:auto,query:(language:kuery,query:''),sort:!('@timestamp',desc))" + +python3 $(readlink -f $(dirname $0))/kibanaSelenium.py diff --git a/tools/gate/selenium/kibanaSelenium.py b/tools/gate/selenium/kibanaSelenium.py new file mode 100644 index 0000000000..8a2d9d06f9 --- /dev/null +++ b/tools/gate/selenium/kibanaSelenium.py @@ -0,0 +1,77 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +from selenium.webdriver.common.by import By +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support import expected_conditions as EC +from selenium.common.exceptions import TimeoutException +from seleniumtester import SeleniumTester + +st = SeleniumTester('Kibana') + +username = st.get_variable('KIBANA_USER') +password = st.get_variable('KIBANA_PASSWORD') +kibana_uri = st.get_variable('KIBANA_URI') +kibana_url = 'http://{0}:{1}@{2}'.format(username, password, kibana_uri) + +try: + st.logger.info('Attempting to connect to Kibana') + st.browser.get(kibana_url) + el = WebDriverWait(st.browser, 45).until( + EC.title_contains('Kibana') + ) + st.logger.info('Connected to Kibana') +except TimeoutException: + st.logger.critical('Timed out waiting for Kibana') + st.browser.quit() + sys.exit(1) + +kernel_query = st.get_variable('KERNEL_QUERY') +journal_query = st.get_variable('JOURNAL_QUERY') +logstash_query = st.get_variable('LOGSTASH_QUERY') + +queries = [(kernel_query, 'Kernel'), + (journal_query, 'Journal'), + (logstash_query, 'Logstash')] + +for query, name in queries: + retry = 3 + while retry > 0: + query_url = '{}/app/kibana#/{}'.format(kibana_url, query) + + try: + st.logger.info('Attempting to query {} index'.format(name)) + st.browser.get(query_url) + WebDriverWait(st.browser, 60).until( + EC.presence_of_element_located( + (By.XPATH, '/html/body/div[2]/div/div/div/div[3]/' + 'discover-app/main/div/div[2]/div/div[2]/section[2]/' + 'doc-table/div/table/tbody/tr[1]/td[2]') + ) + ) + st.logger.info('{} index loaded successfully'.format(name)) + st.take_screenshot('Kibana {} Index'.format(name)) + retry = 0 + + except TimeoutException: + if retry > 1: + st.logger.warning('Timed out loading {} index'.format(name)) + else: + st.logger.error('Could not load {} index'.format(name)) + + retry -= 1 + if retry <= 0: + # Reset test condition + st.browser.get(kibana_url) + +st.browser.quit() diff --git a/tools/gate/selenium/nagios-selenium.sh b/tools/gate/selenium/nagios-selenium.sh new file mode 100755 index 0000000000..d653f1ab12 --- /dev/null +++ b/tools/gate/selenium/nagios-selenium.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +set -xe + +export CHROMEDRIVER="${CHROMEDRIVER:="/etc/selenium/chromedriver"}" +export ARTIFACTS_DIR="${ARTIFACTS_DIR:="/tmp/artifacts/"}" + +export NAGIOS_USER="nagiosadmin" +export NAGIOS_PASSWORD="password" +export NAGIOS_URI="nagios.osh-infra.svc.cluster.local" + +python3 $(readlink -f $(dirname $0))/nagiosSelenium.py diff --git a/tools/gate/selenium/nagiosSelenium.py b/tools/gate/selenium/nagiosSelenium.py new file mode 100755 index 0000000000..c4bf68b5dd --- /dev/null +++ b/tools/gate/selenium/nagiosSelenium.py @@ -0,0 +1,74 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support import expected_conditions as EC +from selenium.common.exceptions import TimeoutException +from selenium.common.exceptions import NoSuchElementException +from seleniumtester import SeleniumTester + +st = SeleniumTester('Nagios') + +username = st.get_variable('NAGIOS_USER') +password = st.get_variable('NAGIOS_PASSWORD') +nagios_uri = st.get_variable('NAGIOS_URI') +nagios_url = 'http://{0}:{1}@{2}'.format(username, password, nagios_uri) + +try: + st.logger.info('Attempting to connect to Nagios') + st.browser.get(nagios_url) + el = WebDriverWait(st.browser, 15).until( + EC.title_contains('Nagios') + ) + st.logger.info('Connected to Nagios') +except TimeoutException: + st.logger.critical('Timed out waiting for Nagios') + st.browser.quit() + sys.exit(1) + +try: + st.logger.info('Switching Focus to Navigation side frame') + sideFrame = st.browser.switch_to.frame('side') +except NoSuchElementException: + st.logger.error('Failed selecting side frame') + st.browser.quit() + sys.exit(1) + +try: + st.logger.info('Attempting to visit Services page') + st.click_link_by_name('Services') + st.take_screenshot('Nagios Services') +except TimeoutException: + st.logger.error('Failed to load Services page') + st.browser.quit() + sys.exit(1) + +try: + st.logger.info('Attempting to visit Host Groups page') + st.click_link_by_name('Host Groups') + st.take_screenshot('Nagios Host Groups') +except TimeoutException: + st.logger.error('Failed to load Host Groups page') + st.browser.quit() + sys.exit(1) + +try: + st.logger.info('Attempting to visit Hosts page') + st.click_link_by_name('Hosts') + st.take_screenshot('Nagios Hosts') +except TimeoutException: + st.logger.error('Failed to load Hosts page') + st.browser.quit() + sys.exit(1) + +st.browser.quit() diff --git a/tools/gate/selenium/prometheus-selenium.sh b/tools/gate/selenium/prometheus-selenium.sh new file mode 100755 index 0000000000..10bb877205 --- /dev/null +++ b/tools/gate/selenium/prometheus-selenium.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +set -xe + +export CHROMEDRIVER="${CHROMEDRIVER:="/etc/selenium/chromedriver"}" +export ARTIFACTS_DIR="${ARTIFACTS_DIR:="/tmp/artifacts/"}" + +export PROMETHEUS_USER="admin" +export PROMETHEUS_PASSWORD="changeme" +export PROMETHEUS_URI="prometheus.osh-infra.svc.cluster.local" + +python3 tools/gate/selenium/prometheusSelenium.py diff --git a/tools/gate/selenium/prometheusSelenium.py b/tools/gate/selenium/prometheusSelenium.py new file mode 100755 index 0000000000..3898f5a3c9 --- /dev/null +++ b/tools/gate/selenium/prometheusSelenium.py @@ -0,0 +1,66 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import sys +from selenium.webdriver.common.by import By +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support import expected_conditions as EC +from selenium.common.exceptions import TimeoutException +from seleniumtester import SeleniumTester + +st = SeleniumTester('Prometheus') + +username = st.get_variable('PROMETHEUS_USER') +password = st.get_variable('PROMETHEUS_PASSWORD') +prometheus_uri = st.get_variable('PROMETHEUS_URI') +prometheus_url = 'http://{}:{}@{}'.format(username, password, prometheus_uri) + +try: + st.logger.info('Attempting to connect to Prometheus') + st.browser.get(prometheus_url) + el = WebDriverWait(st.browser, 15).until( + EC.title_contains('Prometheus') + ) + st.logger.info('Connected to Prometheus') + st.take_screenshot('Prometheus Dashboard') +except TimeoutException: + st.logger.critical('Timed out waiting for Prometheus') + st.browser.quit() + sys.exit(1) + +try: + st.logger.info('Attempting to view Runtime Information') + st.click_link_by_name('Status') + st.click_link_by_name('Runtime & Build Information') + el = WebDriverWait(st.browser, 15).until( + EC.presence_of_element_located((By.XPATH, '/html/body/div/table[1]')) + ) + st.take_screenshot('Prometheus Runtime Info') +except TimeoutException: + st.logger.error('Failed to load Runtime Information page') + st.browser.quit() + sys.exit(1) + +try: + st.logger.info('Attempting to view Runtime Information') + st.click_link_by_name('Status') + st.click_link_by_name('Command-Line Flags') + el = WebDriverWait(st.browser, 15).until( + EC.presence_of_element_located((By.XPATH, '/html/body/div/table')) + ) + st.take_screenshot('Prometheus Command Line Flags') +except TimeoutException: + st.logger.error('Failed to load Command Line Flags page') + st.browser.quit() + sys.exit(1) + +st.browser.quit() diff --git a/tools/gate/selenium/seleniumtester.py b/tools/gate/selenium/seleniumtester.py new file mode 100644 index 0000000000..185a235d25 --- /dev/null +++ b/tools/gate/selenium/seleniumtester.py @@ -0,0 +1,102 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import logging +import sys +from selenium import webdriver +from selenium.webdriver.common.by import By +from selenium.webdriver.support.ui import WebDriverWait +from selenium.webdriver.support import expected_conditions as EC +from selenium.webdriver.chrome.options import Options +from selenium.webdriver.chrome.service import Service +from selenium.common.exceptions import TimeoutException +from selenium.common.exceptions import NoSuchElementException +from selenium.common.exceptions import ScreenshotException + +class SeleniumTester(): + def __init__(self, name): + self.logger = self.get_logger(name) + self.chrome_driver = self.get_variable('CHROMEDRIVER') + self.artifacts_dir = self.get_variable('ARTIFACTS_DIR') + self.initialize_artifiacts_dir() + self.browser = self.get_browser() + + def get_logger(self, name): + logger = logging.getLogger('{} Selenium Tests'.format(name)) + logger.setLevel(logging.DEBUG) + ch = logging.StreamHandler() + ch.setLevel(logging.DEBUG) + formatter = logging.Formatter( + '%(asctime)s - %(name)s - %(levelname)s - %(message)s' + ) + + # Set the formatter and add the handler + ch.setFormatter(formatter) + logger.addHandler(ch) + return logger + + def get_variable(self, env_var): + if env_var in os.environ: + self.logger.info('Found "{}"'.format(env_var)) + return os.environ[env_var] + else: + self.logger.critical( + 'Variable "{}" is not defined!'.format(env_var) + ) + sys.exit(1) + + def get_browser(self): + options = Options() + options.add_argument('--headless') + options.add_argument('--no-sandbox') + options.add_argument('--window-size=1920x1080') + service = Service(executable_path=self.chrome_driver) + browser = webdriver.Chrome(service=service, options=options) + return browser + + def initialize_artifiacts_dir(self): + if self.artifacts_dir and not os.path.exists(self.artifacts_dir): + os.makedirs(self.artifacts_dir) + self.logger.info( + 'Created {} for test artifacts'.format(self.artifacts_dir) + ) + + def click_link_by_name(self, link_name): + try: + el = WebDriverWait(self.browser, 15).until( + EC.presence_of_element_located((By.LINK_TEXT, link_name)) + ) + self.logger.info("Clicking '{}' link".format(link_name)) + link = self.browser.find_element(By.LINK_TEXT, link_name) + link.click() + except (TimeoutException, NoSuchElementException): + self.logger.error("Failed clicking '{}' link".format(link_name)) + self.browser.quit() + sys.exit(1) + + def take_screenshot(self, page_name): + file_name = page_name.replace(' ', '_') + try: + el = WebDriverWait(self.browser, 15) + self.browser.save_screenshot( + '{}{}.png'.format(self.artifacts_dir, file_name) + ) + self.logger.info( + "Successfully captured {} screenshot".format(page_name) + ) + except ScreenshotException: + self.logger.error( + "Failed to capture {} screenshot".format(page_name) + ) + self.browser.quit() + sys.exit(1) diff --git a/tools/pull-images.sh b/tools/pull-images.sh new file mode 100755 index 0000000000..b92ddab682 --- /dev/null +++ b/tools/pull-images.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -x + +if [ "x$1" == "x" ]; then + CHART_DIRS="$(echo ./*/)" +else + CHART_DIRS="$(echo ./$1/)" +fi + +for CHART_DIR in ${CHART_DIRS} ; do + if [ -e ${CHART_DIR}values.yaml ]; then + for IMAGE in $(cat ${CHART_DIR}values.yaml | yq '.images.tags | map(.) | join(" ")' | tr -d '"'); do + sudo docker inspect $IMAGE >/dev/null|| sudo docker pull $IMAGE + done + fi +done diff --git a/values_overrides/ceph-client/apparmor.yaml b/values_overrides/ceph-client/apparmor.yaml new file mode 100644 index 0000000000..21adebd6ce --- /dev/null +++ b/values_overrides/ceph-client/apparmor.yaml @@ -0,0 +1,25 @@ +--- +pod: + mandatory_access_control: + type: apparmor + ceph-checkdns: + ceph-checkdns: runtime/default + init: runtime/default + ceph-mds: + ceph-mds: runtime/default + ceph-init-dirs: runtime/default + ceph-rbd-pool: + ceph-rbd-pool: runtime/default + init: runtime/default + ceph-client-bootstrap: + ceph-client-bootstrap: runtime/default + init: runtime/default + ceph-client-test: + init: runtime/default + ceph-cluster-helm-test: runtime/default +bootstrap: + enabled: true +manifests: + job_bootstrap: true + +... diff --git a/values_overrides/ceph-mon/apparmor.yaml b/values_overrides/ceph-mon/apparmor.yaml new file mode 100644 index 0000000000..fc93e32032 --- /dev/null +++ b/values_overrides/ceph-mon/apparmor.yaml @@ -0,0 +1,39 @@ +--- +pod: + mandatory_access_control: + type: apparmor + ceph-mon: + ceph-init-dirs: runtime/default + ceph-mon: runtime/default + ceph-log-ownership: runtime/default + ceph-mgr: + ceph-mgr: runtime/default + ceph-init-dirs: runtime/default + ceph-mon-check: + ceph-mon: runtime/default + init: runtime/default + ceph-bootstrap: + ceph-bootstrap: runtime/default + init: runtime/default + ceph-storage-keys-generator: + ceph-storage-keys-generator: runtime/default + init: runtime/default + ceph-mon-keyring-generator: + ceph-mon-keyring-generator: runtime/default + init: runtime/default + ceph-mgr-keyring-generator: + init: runtime/default + ceph-mgr-keyring-generator: runtime/default + ceph-mds-keyring-generator: + init: runtime/default + ceph-mds-keyring-generator: runtime/default + ceph-osd-keyring-generator: + ceph-osd-keyring-generator: runtime/default + init: runtime/default + ceph-mon-post-apply: + ceph-mon-post-apply: runtime/default +bootstrap: + enabled: true +manifests: + job_bootstrap: true +... diff --git a/values_overrides/ceph-osd/apparmor.yaml b/values_overrides/ceph-osd/apparmor.yaml new file mode 100644 index 0000000000..36c333a893 --- /dev/null +++ b/values_overrides/ceph-osd/apparmor.yaml @@ -0,0 +1,22 @@ +--- +pod: + mandatory_access_control: + type: apparmor + ceph-osd-default: + ceph-osd-default: runtime/default + log-runner: runtime/default + ceph-init-dirs: runtime/default + ceph-log-ownership: runtime/default + osd-init: runtime/default + init: runtime/default + ceph-osd-test: + init: runtime/default + ceph-cluster-helm-test: runtime/default + ceph-osd-post-apply: + ceph-osd-post-apply: runtime/default + init: runtime/default + lifecycle: + upgrades: + daemonsets: + pod_replacement_strategy: OnDelete +... diff --git a/values_overrides/ceph-provisioners/apparmor.yaml b/values_overrides/ceph-provisioners/apparmor.yaml new file mode 100644 index 0000000000..4ecbe94cfc --- /dev/null +++ b/values_overrides/ceph-provisioners/apparmor.yaml @@ -0,0 +1,31 @@ +--- +pod: + mandatory_access_control: + type: apparmor + ceph-cephfs-client-key-generator: + ceph-storage-keys-generator: runtime/default + init: runtime/default + ceph-rbd-csi-provisioner: + ceph-rbd-provisioner: runtime/default + init: runtime/default + ceph-rbd-snapshotter: runtime/default + ceph-rbd-attacher: runtime/default + csi-resizer: runtime/default + csi-rbdplugin: runtime/default + ceph-provisioner-test: + init: runtime/default + ceph-provisioner-helm-test: runtime/default + ceph-osh-infra-config-test: + init: runtime/default + ceph-provisioner-helm-test: runtime/default + ceph-provisioners-ceph-ns-key-generator: + ceph-storage-keys-generator: runtime/default + init: runtime/default + ceph-rbd-plugin: + driver-registrar: runtime/default + csi-rbdplugin: runtime/default + init: runtime/default + +deployment: + client_secrets: true +... diff --git a/values_overrides/ceph-rgw/2023.1-ubuntu_focal.yaml b/values_overrides/ceph-rgw/2023.1-ubuntu_focal.yaml new file mode 100644 index 0000000000..58e1a7cc0e --- /dev/null +++ b/values_overrides/ceph-rgw/2023.1-ubuntu_focal.yaml @@ -0,0 +1,19 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + ks_endpoints: 'docker.io/openstackhelm/heat:2023.1-ubuntu_focal' + ks_service: 'docker.io/openstackhelm/heat:2023.1-ubuntu_focal' + ks_user: 'docker.io/openstackhelm/heat:2023.1-ubuntu_focal' +... diff --git a/values_overrides/ceph-rgw/2024.1-ubuntu_jammy.yaml b/values_overrides/ceph-rgw/2024.1-ubuntu_jammy.yaml new file mode 100644 index 0000000000..1acc2b9df9 --- /dev/null +++ b/values_overrides/ceph-rgw/2024.1-ubuntu_jammy.yaml @@ -0,0 +1,19 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + ks_endpoints: 'docker.io/openstackhelm/heat:2024.1-ubuntu_jammy' + ks_service: 'docker.io/openstackhelm/heat:2024.1-ubuntu_jammy' + ks_user: 'docker.io/openstackhelm/heat:2024.1-ubuntu_jammy' +... diff --git a/values_overrides/ceph-rgw/2024.2-ubuntu_jammy.yaml b/values_overrides/ceph-rgw/2024.2-ubuntu_jammy.yaml new file mode 100644 index 0000000000..087ae6b90a --- /dev/null +++ b/values_overrides/ceph-rgw/2024.2-ubuntu_jammy.yaml @@ -0,0 +1,19 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + ks_endpoints: 'docker.io/openstackhelm/heat:2024.2-ubuntu_jammy' + ks_service: 'docker.io/openstackhelm/heat:2024.2-ubuntu_jammy' + ks_user: 'docker.io/openstackhelm/heat:2024.2-ubuntu_jammy' +... diff --git a/values_overrides/ceph-rgw/apparmor.yaml b/values_overrides/ceph-rgw/apparmor.yaml new file mode 100644 index 0000000000..be6935f748 --- /dev/null +++ b/values_overrides/ceph-rgw/apparmor.yaml @@ -0,0 +1,35 @@ +--- +pod: + mandatory_access_control: + type: apparmor + ceph-rgw: + init: runtime/default + ceph-rgw: runtime/default + ceph-init-dirs: runtime/default + ceph-rgw-init: runtime/default + ceph-rgw-bootstrap: + ceph-keyring-placement: runtime/default + init: runtime/default + ceph-rgw-bootstrap: runtime/default + ceph-rgw-storage-init: + ceph-keyring-placement: runtime/default + init: runtime/default + ceph-rgw-storage-init: runtime/default + ceph-rgw-s3-admin: + ceph-keyring-placement: runtime/default + init: runtime/default + create-s3-admin: runtime/default + ceph-rgw-pool: + ceph-rgw-pool: runtime/default + init: runtime/default + ceph-rgw-test: + ceph-rgw-ks-validation: runtime/default + ceph-rgw-s3-validation: runtime/default +conf: + rgw_s3: + enabled: true +bootstrap: + enabled: true +manifests: + job_bootstrap: true +... diff --git a/values_overrides/ceph-rgw/netpol.yaml b/values_overrides/ceph-rgw/netpol.yaml new file mode 100644 index 0000000000..958a2b4d0b --- /dev/null +++ b/values_overrides/ceph-rgw/netpol.yaml @@ -0,0 +1,22 @@ +--- +manifests: + network_policy: true +network_policy: + rgw: + egress: + - to: + - ipBlock: + cidr: 172.17.0.1/16 + - to: + ports: + - protocol: TCP + port: 80 + - protocol: TCP + port: 443 + - to: + - ipBlock: + cidr: %%%REPLACE_API_ADDR%%%/32 + ports: + - protocol: TCP + port: %%%REPLACE_API_PORT%%% +... diff --git a/values_overrides/ceph-rgw/tls.yaml b/values_overrides/ceph-rgw/tls.yaml new file mode 100644 index 0000000000..6f708d3d3c --- /dev/null +++ b/values_overrides/ceph-rgw/tls.yaml @@ -0,0 +1,45 @@ +--- +endpoints: + object_store: + scheme: + default: https + host_fqdn_override: + default: + tls: + secretName: ceph-rgw-ks-tls-api + issuerRef: + name: ca-issuer + kind: ClusterIssuer + ceph_object_store: + scheme: + default: https + host_fqdn_override: + default: + tls: + secretName: ceph-rgw-s3-tls-api + issuerRef: + name: ca-issuer + kind: ClusterIssuer + +network: + api: + ingress: + public: true + classes: + namespace: "nginx" + cluster: "nginx-cluster" + annotations: + nginx.ingress.kubernetes.io/rewrite-target: / + nginx.ingress.kubernetes.io/proxy-body-size: "0" + nginx.ingress.kubernetes.io/proxy-max-temp-file-size: "0" + nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" + external_policy_local: false + node_port: + enabled: false + port: 30004 + public: 192.168.0.0/16 + cluster: 192.168.0.0/16 + +manifests: + certificates: true +... diff --git a/values_overrides/daemonjob-controller/apparmor.yaml b/values_overrides/daemonjob-controller/apparmor.yaml new file mode 100644 index 0000000000..622db08e02 --- /dev/null +++ b/values_overrides/daemonjob-controller/apparmor.yaml @@ -0,0 +1,7 @@ +--- +pod: + mandatory_access_control: + type: apparmor + daemonjob-controller: + controller: runtime/default +... diff --git a/values_overrides/elastic-apm-server/apparmor.yaml b/values_overrides/elastic-apm-server/apparmor.yaml new file mode 100644 index 0000000000..70b0988d79 --- /dev/null +++ b/values_overrides/elastic-apm-server/apparmor.yaml @@ -0,0 +1,8 @@ +--- +pod: + mandatory_access_control: + type: apparmor + elastic-apm-server: + init: runtime/default + elastic-apm-server: runtime/default +... diff --git a/values_overrides/elastic-filebeat/apparmor.yaml b/values_overrides/elastic-filebeat/apparmor.yaml new file mode 100644 index 0000000000..6f65ccd73e --- /dev/null +++ b/values_overrides/elastic-filebeat/apparmor.yaml @@ -0,0 +1,8 @@ +--- +pod: + mandatory_access_control: + type: apparmor + filebeat: + filebeat: runtime/default + init: runtime/default +... diff --git a/values_overrides/elasticsearch/2023.1-ubuntu_focal.yaml b/values_overrides/elasticsearch/2023.1-ubuntu_focal.yaml new file mode 100644 index 0000000000..28c5284856 --- /dev/null +++ b/values_overrides/elasticsearch/2023.1-ubuntu_focal.yaml @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + memory_init: docker.io/openstackhelm/heat:2023.1-ubuntu_focal + helm_tests: docker.io/openstackhelm/heat:2023.1-ubuntu_focal +... diff --git a/values_overrides/elasticsearch/2024.1-ubuntu_jammy.yaml b/values_overrides/elasticsearch/2024.1-ubuntu_jammy.yaml new file mode 100644 index 0000000000..908e3de41f --- /dev/null +++ b/values_overrides/elasticsearch/2024.1-ubuntu_jammy.yaml @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + memory_init: docker.io/openstackhelm/heat:2024.1-ubuntu_jammy + helm_tests: docker.io/openstackhelm/heat:2024.1-ubuntu_jammy +... diff --git a/values_overrides/elasticsearch/2024.2-ubuntu_jammy.yaml b/values_overrides/elasticsearch/2024.2-ubuntu_jammy.yaml new file mode 100644 index 0000000000..cecc859f32 --- /dev/null +++ b/values_overrides/elasticsearch/2024.2-ubuntu_jammy.yaml @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + memory_init: docker.io/openstackhelm/heat:2024.2-ubuntu_jammy + helm_tests: docker.io/openstackhelm/heat:2024.2-ubuntu_jammy +... diff --git a/values_overrides/elasticsearch/apparmor.yaml b/values_overrides/elasticsearch/apparmor.yaml new file mode 100644 index 0000000000..4504195dec --- /dev/null +++ b/values_overrides/elasticsearch/apparmor.yaml @@ -0,0 +1,34 @@ +--- +pod: + env: + client: null + data: null + master: null + mandatory_access_control: + type: apparmor + elasticsearch-master: + elasticsearch-master: runtime/default + init: runtime/default + memory-map-increase: runtime/default + elasticsearch-data: + elasticsearch-data: runtime/default + init: runtime/default + memory-map-increase: runtime/default + elasticsearch-client: + elasticsearch-client: runtime/default + init: runtime/default + memory-map-increase: runtime/default + apache-proxy: runtime/default + prometheus-elasticsearch-exporter: + elasticsearch-exporter: runtime/default + init: runtime/default + elasticsearch-test: + init: runtime/default + elasticsearch-helm-tests: runtime/default + create-elasticsearch-templates: + create-elasticsearch-templates: runtime/default + init: runtime/default + elasticsearch-verify-repositories: + elasticsearch-verify-repositories: runtime/default + init: runtime/default +... diff --git a/values_overrides/elasticsearch/local-storage.yaml b/values_overrides/elasticsearch/local-storage.yaml new file mode 100644 index 0000000000..8219609e99 --- /dev/null +++ b/values_overrides/elasticsearch/local-storage.yaml @@ -0,0 +1,22 @@ +--- +pod: + replicas: + data: 1 +storage: + data: + requests: + storage: 1Gi + storage_class: local-storage + master: + requests: + storage: 1Gi + storage_class: local-storage +manifests: + cron_curator: false + cron_verify_repositories: false + job_snapshot_repository: false + job_elasticsearch_templates: false + job_s3_user: false + job_s3_bucket: false + helm_tests: false +... diff --git a/values_overrides/elasticsearch/remote-cluster.yaml b/values_overrides/elasticsearch/remote-cluster.yaml new file mode 100644 index 0000000000..ca00971ed8 --- /dev/null +++ b/values_overrides/elasticsearch/remote-cluster.yaml @@ -0,0 +1,32 @@ +# Can't use these settings at startup yet becuse of +# https://github.com/elastic/elasticsearch/issues/27006 +# conf: +# elasticsearch: +# config: +# cluster: +# remote: +# remote_elasticsearch: +# seeds: +# - elasticsearch-gateway-1.remote_host:9301 +# - elasticsearch-gateway-2.remote_host:9301 +# - elasticsearch-gateway-3.remote_host:9301 +# skip_unavailale: true +--- +network: + remote_clustering: + enabled: true + +manifests: + cron_curator: false + cron_verify_repositories: false + job_snapshot_repository: false +pod: + replicas: + master: 2 + data: 1 + client: 1 + gateway: 1 +images: + tags: + elasticsearch: docker.io/openstackhelm/elasticsearch-s3:7_6_2-centos_7 +... diff --git a/values_overrides/elasticsearch/tls.yaml b/values_overrides/elasticsearch/tls.yaml new file mode 100644 index 0000000000..ed684c9415 --- /dev/null +++ b/values_overrides/elasticsearch/tls.yaml @@ -0,0 +1,156 @@ +--- +endpoints: + elasticsearch: + host_fqdn_override: + default: + tls: + secretName: elasticsearch-tls-api + issuerRef: + name: ca-issuer + kind: ClusterIssuer + scheme: + default: "https" + port: + http: + default: 443 +network: + elasticsearch: + ingress: + annotations: + nginx.ingress.kubernetes.io/backend-protocol: https +conf: + httpd: | + ServerRoot "/usr/local/apache2" + + Listen 443 + + LoadModule allowmethods_module modules/mod_allowmethods.so + LoadModule mpm_event_module modules/mod_mpm_event.so + LoadModule authn_file_module modules/mod_authn_file.so + LoadModule authn_core_module modules/mod_authn_core.so + LoadModule authz_host_module modules/mod_authz_host.so + LoadModule authz_groupfile_module modules/mod_authz_groupfile.so + LoadModule authz_user_module modules/mod_authz_user.so + LoadModule authz_core_module modules/mod_authz_core.so + LoadModule access_compat_module modules/mod_access_compat.so + LoadModule auth_basic_module modules/mod_auth_basic.so + LoadModule ldap_module modules/mod_ldap.so + LoadModule authnz_ldap_module modules/mod_authnz_ldap.so + LoadModule reqtimeout_module modules/mod_reqtimeout.so + LoadModule filter_module modules/mod_filter.so + LoadModule proxy_html_module modules/mod_proxy_html.so + LoadModule log_config_module modules/mod_log_config.so + LoadModule env_module modules/mod_env.so + LoadModule headers_module modules/mod_headers.so + LoadModule setenvif_module modules/mod_setenvif.so + LoadModule version_module modules/mod_version.so + LoadModule proxy_module modules/mod_proxy.so + LoadModule proxy_connect_module modules/mod_proxy_connect.so + LoadModule proxy_http_module modules/mod_proxy_http.so + LoadModule proxy_balancer_module modules/mod_proxy_balancer.so + LoadModule slotmem_shm_module modules/mod_slotmem_shm.so + LoadModule slotmem_plain_module modules/mod_slotmem_plain.so + LoadModule unixd_module modules/mod_unixd.so + LoadModule status_module modules/mod_status.so + LoadModule autoindex_module modules/mod_autoindex.so + LoadModule rewrite_module modules/mod_rewrite.so + LoadModule ssl_module modules/mod_ssl.so + + + User daemon + Group daemon + + + + AllowOverride none + Require all denied + + + + Require all denied + + + ErrorLog /dev/stderr + + LogLevel warn + + + LogFormat "%a %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined + LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" proxy + LogFormat "%h %l %u %t \"%r\" %>s %b" common + + + LogFormat "%a %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combinedio + + + SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded + CustomLog /dev/stdout common + CustomLog /dev/stdout combined + CustomLog /dev/stdout proxy env=forwarded + + + + AllowOverride None + Options None + Require all granted + + + + RequestHeader unset Proxy early + + + + Include conf/extra/proxy-html.conf + + + + + ProxyPass http://localhost:{{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/ + ProxyPassReverse http://localhost:{{ tuple "elasticsearch" "internal" "client" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/ + AuthName "Elasticsearch" + AuthType Basic + AuthBasicProvider file ldap + AuthUserFile /usr/local/apache2/conf/.htpasswd + AuthLDAPBindDN {{ .Values.endpoints.ldap.auth.admin.bind }} + AuthLDAPBindPassword {{ .Values.endpoints.ldap.auth.admin.password }} + AuthLDAPURL {{ tuple "ldap" "default" "ldap" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | quote }} + Require valid-user + + + # Restrict access to the Elasticsearch Update By Query API Endpoint to prevent modification of indexed documents + + Require all denied + + # Restrict access to the Elasticsearch Delete By Query API Endpoint to prevent deletion of indexed documents + + Require all denied + + SSLEngine On + SSLProxyEngine on + SSLCertificateFile /etc/elasticsearch/certs/tls.crt + SSLCertificateKeyFile /etc/elasticsearch/certs/tls.key + SSLProtocol all -SSLv3 -TLSv1 -TLSv1.1 + SSLCipherSuite ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256 + SSLHonorCipherOrder on + + elasticsearch: + config: + xpack: + security: + enabled: true + transport: + ssl: + enabled: true + verification_mode: certificate + key: /usr/share/elasticsearch/config/tls.key + certificate: /usr/share/elasticsearch/config/tls.crt + certificate_authorities: ["/usr/share/elasticsearch/config/ca.crt"] + curator: + config: + client: + use_ssl: True + ssl_no_validate: False + certificate: '/etc/elasticsearch/certs/ca.crt' +manifests: + certificates: true +... diff --git a/values_overrides/fluentd/2023.1-ubuntu_focal.yaml b/values_overrides/fluentd/2023.1-ubuntu_focal.yaml new file mode 100644 index 0000000000..1292734fc6 --- /dev/null +++ b/values_overrides/fluentd/2023.1-ubuntu_focal.yaml @@ -0,0 +1,17 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + helm_tests: docker.io/openstackhelm/heat:2023.1-ubuntu_focal +... diff --git a/values_overrides/fluentd/2024.1-ubuntu_jammy.yaml b/values_overrides/fluentd/2024.1-ubuntu_jammy.yaml new file mode 100644 index 0000000000..efba1791d5 --- /dev/null +++ b/values_overrides/fluentd/2024.1-ubuntu_jammy.yaml @@ -0,0 +1,17 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + helm_tests: docker.io/openstackhelm/heat:2024.1-ubuntu_jammy +... diff --git a/values_overrides/fluentd/2024.2-ubuntu_jammy.yaml b/values_overrides/fluentd/2024.2-ubuntu_jammy.yaml new file mode 100644 index 0000000000..d389163c67 --- /dev/null +++ b/values_overrides/fluentd/2024.2-ubuntu_jammy.yaml @@ -0,0 +1,17 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + helm_tests: docker.io/openstackhelm/heat:2024.2-ubuntu_jammy +... diff --git a/values_overrides/fluentd/apparmor.yaml b/values_overrides/fluentd/apparmor.yaml new file mode 100644 index 0000000000..b5121b5821 --- /dev/null +++ b/values_overrides/fluentd/apparmor.yaml @@ -0,0 +1,8 @@ +--- +pod: + mandatory_access_control: + type: apparmor + fluentd: + fluentd: runtime/default + init: runtime/default +... diff --git a/values_overrides/fluentd/tls.yaml b/values_overrides/fluentd/tls.yaml new file mode 100644 index 0000000000..10575b8435 --- /dev/null +++ b/values_overrides/fluentd/tls.yaml @@ -0,0 +1,41 @@ +--- +conf: + fluentd: + conf: + output: | + +endpoints: + elasticsearch: + scheme: + default: "https" + port: + http: + default: 443 +manifests: + certificates: true +... diff --git a/values_overrides/gnocchi/2023.2-ubuntu-jammy.yaml b/values_overrides/gnocchi/2023.2-ubuntu-jammy.yaml new file mode 100644 index 0000000000..ff4fe61a81 --- /dev/null +++ b/values_overrides/gnocchi/2023.2-ubuntu-jammy.yaml @@ -0,0 +1,37 @@ +--- +images: + tags: + db_init: quay.io/openstack.kolla/gnocchi-api:2023.2-ubuntu-jammy + db_sync: quay.io/openstack.kolla/gnocchi-api:2023.2-ubuntu-jammy + ks_user: docker.io/openstackhelm/heat:2023.2-ubuntu_jammy + ks_service: docker.io/openstackhelm/heat:2023.2-ubuntu_jammy + ks_endpoints: docker.io/openstackhelm/heat:2023.2-ubuntu_jammy + gnocchi_api: quay.io/openstack.kolla/gnocchi-api:2023.2-ubuntu-jammy + gnocchi_statsd: quay.io/openstack.kolla/gnocchi-statsd:2023.2-ubuntu-jammy + gnocchi_metricd: quay.io/openstack.kolla/gnocchi-metricd:2023.2-ubuntu-jammy + gnocchi_resources_cleaner: quay.io/openstack.kolla/gnocchi-base:2023.2-ubuntu-jammy +conf: + apache: | + Listen 0.0.0.0:{{ tuple "metric" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }} + + SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded + CustomLog /dev/stdout combined env=!forwarded + CustomLog /dev/stdout proxy env=forwarded + + + WSGIDaemonProcess gnocchi processes=1 threads=2 user=gnocchi group=gnocchi display-name=%{GROUP} + WSGIProcessGroup gnocchi + WSGIScriptAlias / "/var/lib/kolla/venv/bin/gnocchi-api" + WSGIApplicationGroup %{GLOBAL} + + ErrorLog /dev/stderr + SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded + CustomLog /dev/stdout combined env=!forwarded + CustomLog /dev/stdout proxy env=forwarded + + + Require all granted + + + enable_paste: False +... diff --git a/values_overrides/grafana/2024.1-ubuntu_jammy.yaml b/values_overrides/grafana/2024.1-ubuntu_jammy.yaml new file mode 100644 index 0000000000..124f3b2ae6 --- /dev/null +++ b/values_overrides/grafana/2024.1-ubuntu_jammy.yaml @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + db_init: docker.io/openstackhelm/heat:2024.1-ubuntu_jammy + grafana_db_session_sync: docker.io/openstackhelm/heat:2024.1-ubuntu_jammy +... diff --git a/values_overrides/grafana/2024.2-ubuntu_jammy.yaml b/values_overrides/grafana/2024.2-ubuntu_jammy.yaml new file mode 100644 index 0000000000..af7dc51c95 --- /dev/null +++ b/values_overrides/grafana/2024.2-ubuntu_jammy.yaml @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + db_init: docker.io/openstackhelm/heat:2024.2-ubuntu_jammy + grafana_db_session_sync: docker.io/openstackhelm/heat:2024.2-ubuntu_jammy +... diff --git a/values_overrides/grafana/apparmor.yaml b/values_overrides/grafana/apparmor.yaml new file mode 100644 index 0000000000..4693d2929e --- /dev/null +++ b/values_overrides/grafana/apparmor.yaml @@ -0,0 +1,27 @@ +--- +pod: + mandatory_access_control: + type: apparmor + grafana: + grafana: runtime/default + init: runtime/default + grafana-db-init-session: + grafana-db-init-session: runtime/default + init: runtime/default + grafana-db-init: + grafana-db-init: runtime/default + init: runtime/default + grafana-db-session-sync: + grafana-db-session-sync: runtime/default + init: runtime/default + grafana-set-admin-user: + grafana-set-admin-password: runtime/default + init: runtime/default + grafana-run-migrator: + grafana-run-migrator: runtime/default + prepare-grafana-migrator: runtime/default + init: runtime/default + grafana-test: + init: runtime/default + grafana-selenium-tests: runtime/default +... diff --git a/values_overrides/grafana/calico.yaml b/values_overrides/grafana/calico.yaml new file mode 100644 index 0000000000..44741d55ed --- /dev/null +++ b/values_overrides/grafana/calico.yaml @@ -0,0 +1,1362 @@ +# NOTE(srwilkers): This overrides file provides a reference for a dashboard for +# the Calico CNI +--- +conf: + dashboards: + network: + calico: |- + { + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "5.0.0" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Calico cluster monitoring dashboard", + "overwrite": true, + "editable": false, + "gnetId": 3244, + "graphTooltip": 0, + "id": 38, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "true": 0, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 15, + "panels": [], + "repeat": null, + "title": "Felix", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fill": 1, + "gridPos": { + "h": 7, + "true": 1, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 1, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "felix_active_local_endpoints", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 20 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Active Local Endpoints", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fill": 1, + "gridPos": { + "h": 7, + "true": 1, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "felix_active_local_policies", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 20 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Active Local Policies", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fill": 1, + "gridPos": { + "h": 7, + "true": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "id": 2, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "felix_active_local_selectors", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 20 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Active Local Selectors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fill": 1, + "gridPos": { + "h": 7, + "true": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "felix_active_local_tags", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 20 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Active Local Tags", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fill": 1, + "gridPos": { + "h": 7, + "true": 15, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 5, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "felix_cluster_num_host_endpoints", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 20 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cluster Host Endpoints", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fill": 1, + "gridPos": { + "h": 7, + "true": 15, + "w": 12, + "x": 12, + "y": 15 + }, + "id": 6, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "felix_cluster_num_workload_endpoints", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 20 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cluster Workload Endpoints", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fill": 1, + "gridPos": { + "h": 7, + "true": 22, + "w": 12, + "x": 0, + "y": 22 + }, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "felix_cluster_num_hosts", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 20 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Clusters Hosts", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fill": 1, + "gridPos": { + "h": 7, + "true": 22, + "w": 12, + "x": 12, + "y": 22 + }, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "felix_ipsets_calico", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 20 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Active IP Sets", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fill": 1, + "gridPos": { + "h": 7, + "true": 29, + "w": 12, + "x": 0, + "y": 29 + }, + "id": 9, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "felix_iptables_chains", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 20 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Active IP Tables Chains", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fill": 1, + "gridPos": { + "h": 7, + "true": 29, + "w": 12, + "x": 12, + "y": 29 + }, + "id": 10, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "felix_ipset_errors", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 20 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "IP Set Command Failures", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fill": 1, + "gridPos": { + "h": 7, + "true": 36, + "w": 12, + "x": 0, + "y": 36 + }, + "id": 11, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "felix_iptables_save_errors", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 20 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "IP Tables Save Errors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fill": 1, + "gridPos": { + "h": 7, + "true": 36, + "w": 12, + "x": 12, + "y": 36 + }, + "id": 12, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "felix_iptables_restore_errors", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 20 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "IP Tables Restore Errors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fill": 1, + "gridPos": { + "h": 7, + "true": 43, + "w": 12, + "x": 0, + "y": 43 + }, + "id": 13, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "felix_resyncs_started", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 20 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Felix Resyncing Datastore", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fill": 1, + "gridPos": { + "h": 7, + "true": 43, + "w": 12, + "x": 12, + "y": 43 + }, + "id": 14, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "felix_int_dataplane_failures", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A", + "step": 20 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Dataplane failed updates", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "refresh": "5m", + "schemaVersion": 18, + "style": "dark", + "tags": [ + "calico" + ], + "templating": { + "list": [ + { + "current": { + "text": "prometheus", + "value": "prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Prometheus datasource", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Kubernetes Calico", + "version": 1 + } +... diff --git a/values_overrides/grafana/ceph.yaml b/values_overrides/grafana/ceph.yaml new file mode 100644 index 0000000000..87e53ccf6a --- /dev/null +++ b/values_overrides/grafana/ceph.yaml @@ -0,0 +1,3677 @@ +# NOTE(srwilkers): This overrides file provides a reference for dashboards for +# the overall state of ceph clusters, ceph osds in those clusters, and the +# status of ceph pools for those clusters +--- +conf: + dashboards: + ceph: + ceph_cluster: |- + { + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "prometheus", + "description": "Prometheus.IO", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "3.1.1" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "singlestat", + "name": "Singlestat", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Ceph Cluster overview.\r\n", + "overwrite": true, + "editable": false, + "gnetId": 917, + "graphTooltip": 0, + "id": 14, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 35, + "panels": [], + "title": "New row", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 21, + "interval": "1m", + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "ceph_health_status{application=\"ceph\",release_group=\"$ceph_cluster\"}", + "interval": "$interval", + "intervalFactor": 1, + "refId": "A", + "step": 60 + } + ], + "thresholds": "1,1", + "title": "Status", + "type": "singlestat", + "valueFontSize": "100%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + }, + { + "op": "=", + "text": "HEALTHY", + "value": "0" + }, + { + "op": "=", + "text": "WARNING", + "value": "1" + }, + { + "op": "=", + "text": "CRITICAL", + "value": "2" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 22, + "interval": "1m", + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "count(ceph_pool_max_avail{application=\"ceph\",release_group=\"$ceph_cluster\"})", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 60 + } + ], + "thresholds": "", + "title": "Pools", + "type": "singlestat", + "valueFontSize": "100%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 1 + }, + "id": 33, + "interval": "1m", + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "ceph_cluster_total_bytes{application=\"ceph\",release_group=\"$ceph_cluster\"}", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 60 + } + ], + "thresholds": "0.025,0.1", + "title": "Cluster Capacity", + "type": "singlestat", + "valueFontSize": "100%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 12, + "y": 1 + }, + "id": 34, + "interval": "1m", + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "ceph_cluster_total_used_bytes{application=\"ceph\",release_group=\"$ceph_cluster\"}", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 60 + } + ], + "thresholds": "0.025,0.1", + "title": "Used Capacity", + "type": "singlestat", + "valueFontSize": "100%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 16, + "y": 1 + }, + "id": 23, + "interval": "1m", + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "ceph_cluster_total_used_bytes/ceph_cluster_total_bytes{application=\"ceph\",release_group=\"$ceph_cluster\"}", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 60 + } + ], + "thresholds": "70,80", + "title": "Current Utilization", + "type": "singlestat", + "valueFontSize": "100%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 36, + "panels": [], + "title": "New row", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 0, + "y": 6 + }, + "id": 26, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(ceph_osd_in{application=\"ceph\",release_group=\"$ceph_cluster\"})", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 60 + } + ], + "thresholds": "", + "title": "OSDs IN", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 40, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 2, + "y": 6 + }, + "id": 27, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(ceph_osd_metadata{application=\"ceph\",release_group=\"$ceph_cluster\"}) - sum(ceph_osd_in{application=\"ceph\",release_group=\"$ceph_cluster\"})", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 60 + } + ], + "thresholds": "1,1", + "title": "OSDs OUT", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 4, + "y": 6 + }, + "id": 28, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(ceph_osd_up{application=\"ceph\",release_group=\"$ceph_cluster\"})", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 60 + } + ], + "thresholds": "", + "title": "OSDs UP", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 40, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 6, + "y": 6 + }, + "id": 29, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(ceph_osd_metadata{application=\"ceph\",release_group=\"$ceph_cluster\"}) - sum(ceph_osd_up{application=\"ceph\",release_group=\"$ceph_cluster\"})", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 60 + } + ], + "thresholds": "1,1", + "title": "OSDs DOWN", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 6 + }, + "id": 30, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "avg(ceph_osd_numpg{application=\"ceph\",release_group=\"$ceph_cluster\"})", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 60 + } + ], + "thresholds": "250,300", + "title": "Average PGs per OSD", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 37, + "panels": [], + "repeat": null, + "title": "CLUSTER", + "type": "row" + }, + { + "aliasColors": { + "Available": "#EAB839", + "Total Capacity": "#447EBC", + "Used": "#BF1B00", + "total_avail": "#6ED0E0", + "total_space": "#7EB26D", + "total_used": "#890F02" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 4, + "grid": {}, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 10 + }, + "height": "300", + "id": 1, + "interval": "$interval", + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Total Capacity", + "fill": 0, + "linewidth": 3, + "stack": false + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceph_cluster_total_bytes{application=\"ceph\",release_group=\"$ceph_cluster\"} - ceph_cluster_total_used_bytes{application=\"ceph\",release_group=\"$ceph_cluster\"}", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Available", + "refId": "A", + "step": 60 + }, + { + "expr": "ceph_cluster_total_used_bytes", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Used", + "refId": "B", + "step": 60 + }, + { + "expr": "ceph_cluster_total_bytes", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Total Capacity", + "refId": "C", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Capacity", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Total Capacity": "#7EB26D", + "Used": "#BF1B00", + "total_avail": "#6ED0E0", + "total_space": "#7EB26D", + "total_used": "#890F02" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 0, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 10 + }, + "height": "300", + "id": 3, + "interval": "$interval", + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(ceph_osd_op_w{application=\"ceph\",release_group=\"$ceph_cluster\"})", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Write", + "refId": "A", + "step": 60 + }, + { + "expr": "sum(ceph_osd_op_r{application=\"ceph\",release_group=\"$ceph_cluster\"})", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Read", + "refId": "B", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "IOPS", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": "", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 10 + }, + "height": "300", + "id": 7, + "interval": "$interval", + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(ceph_osd_op_in_bytes{application=\"ceph\",release_group=\"$ceph_cluster\"})", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Write", + "refId": "A", + "step": 60 + }, + { + "expr": "sum(ceph_osd_op_out_bytes{application=\"ceph\",release_group=\"$ceph_cluster\"})", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Read", + "refId": "B", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Throughput", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 18 + }, + "id": 38, + "panels": [], + "title": "New row", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 18, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^Total.*$/", + "stack": false + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceph_cluster_total_objects{application=\"ceph\",release_group=\"$ceph_cluster\"}", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Total", + "refId": "A", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Objects in the Cluster", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 19, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^Total.*$/", + "stack": false + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(ceph_osd_numpg{application=\"ceph\",release_group=\"$ceph_cluster\"})", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Total", + "refId": "A", + "step": 60 + }, + { + "expr": "sum(ceph_pg_active{application=\"ceph\",release_group=\"$ceph_cluster\"})", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Active", + "refId": "B", + "step": 60 + }, + { + "expr": "sum(ceph_pg_inconsistent{application=\"ceph\",release_group=\"$ceph_cluster\"})", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Inconsistent", + "refId": "C", + "step": 60 + }, + { + "expr": "sum(ceph_pg_creating{application=\"ceph\",release_group=\"$ceph_cluster\"})", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Creating", + "refId": "D", + "step": 60 + }, + { + "expr": "sum(ceph_pg_recovering{application=\"ceph\",release_group=\"$ceph_cluster\"})", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Recovering", + "refId": "E", + "step": 60 + }, + { + "expr": "sum(ceph_pg_down{application=\"ceph\",release_group=\"$ceph_cluster\"})", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Down", + "refId": "F", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "PGs", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 20, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^Total.*$/", + "stack": false + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(ceph_pg_degraded{application=\"ceph\",release_group=\"$ceph_cluster\"})", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Degraded", + "refId": "A", + "step": 60 + }, + { + "expr": "sum(ceph_pg_stale{application=\"ceph\",release_group=\"$ceph_cluster\"})", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Stale", + "refId": "B", + "step": 60 + }, + { + "expr": "sum(ceph_pg_undersized{application=\"ceph\",release_group=\"$ceph_cluster\"})", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Undersized", + "refId": "C", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Stuck PGs", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 1, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "5m", + "schemaVersion": 18, + "style": "dark", + "tags": [ + "ceph", + "cluster" + ], + "templating": { + "list": [ + { + "current": { + "text": "prometheus", + "value": "prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Prometheus datasource", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "", + "hide": 0, + "includeAll": false, + "label": "Cluster", + "multi": false, + "name": "ceph_cluster", + "options": [], + "query": "label_values(ceph_health_status, release_group)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "auto": true, + "auto_count": 10, + "auto_min": "1m", + "current": { + "text": "1m", + "value": "1m" + }, + "datasource": null, + "hide": 0, + "includeAll": false, + "label": "Interval", + "multi": false, + "name": "interval", + "options": [ + { + "selected": false, + "text": "auto", + "value": "$__auto_interval_interval" + }, + { + "selected": true, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Ceph - Cluster", + "version": 1 + } + ceph_osd: |- + { + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "prometheus", + "description": "Prometheus.IO", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "3.1.1" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "singlestat", + "name": "Singlestat", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "CEPH OSD Status.", + "overwrite": true, + "editable": true, + "gnetId": 923, + "graphTooltip": 0, + "id": 17, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 11, + "panels": [], + "title": "New row", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 40, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 0, + "y": 1 + }, + "id": 6, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 2, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + }, + { + "from": "0", + "text": "DOWN", + "to": "0.99" + }, + { + "from": "0.99", + "text": "UP", + "to": "1" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "ceph_osd_up{ceph_daemon=\"$osd\",application=\"ceph\",release_group=\"$ceph_cluster\"}", + "interval": "$interval", + "intervalFactor": 1, + "refId": "A", + "step": 60 + } + ], + "thresholds": "0,1", + "timeFrom": null, + "title": "Status", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "DOWN", + "value": "0" + }, + { + "op": "=", + "text": "UP", + "value": "1" + }, + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 40, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 2, + "y": 1 + }, + "id": 8, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 2, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + }, + { + "from": "0", + "text": "OUT", + "to": "0.99" + }, + { + "from": "0.99", + "text": "IN", + "to": "1" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "ceph_osd_in{ceph_daemon=\"$osd\",application=\"ceph\",release_group=\"$ceph_cluster\"}", + "interval": "$interval", + "intervalFactor": 1, + "refId": "A", + "step": 60 + } + ], + "thresholds": "0,1", + "timeFrom": null, + "title": "Available", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "DOWN", + "value": "0" + }, + { + "op": "=", + "text": "UP", + "value": "1" + }, + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 2, + "x": 4, + "y": 1 + }, + "id": 10, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 2, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "count(ceph_osd_metadata{application=\"ceph\",release_group=\"$ceph_cluster\"})", + "interval": "$interval", + "intervalFactor": 1, + "refId": "A", + "step": 60 + } + ], + "thresholds": "0,1", + "timeFrom": null, + "title": "Total OSDs", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "DOWN", + "value": "0" + }, + { + "op": "=", + "text": "UP", + "value": "1" + }, + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 12, + "panels": [], + "title": "OSD: $osd", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 20, + "x": 0, + "y": 5 + }, + "id": 5, + "interval": "$interval", + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^Average.*/", + "fill": 0, + "stack": false + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceph_osd_numpg{ceph_daemon=~\"$osd\",application=\"ceph\",release_group=\"$ceph_cluster\"}", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Number of PGs - {{ $osd }}", + "refId": "A", + "step": 60 + }, + { + "expr": "avg(ceph_osd_numpg{application=\"ceph\",release_group=\"$ceph_cluster\"})", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Average Number of PGs in the Cluster", + "refId": "B", + "step": 60 + } + ], + "thresholds": [ + { + "colorMode": "custom", + "line": true, + "lineColor": "rgba(216, 200, 27, 0.27)", + "op": "gt", + "value": 250 + }, + { + "colorMode": "custom", + "line": true, + "lineColor": "rgba(234, 112, 112, 0.22)", + "op": "gt", + "value": 300 + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "PGs", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 20, + "y": 5 + }, + "id": 7, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "(ceph_osd_stat_bytes_used{ceph_daemon=~\"$osd\",application=\"ceph\",release_group=\"$ceph_cluster\"}/ceph_osd_stat_bytes{ceph_daemon=~\"$osd\",application=\"ceph\",release_group=\"$ceph_cluster\"})*100", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A", + "step": 60 + } + ], + "thresholds": "60,80", + "timeFrom": null, + "title": "Utilization", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 13, + "panels": [], + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 13 + }, + "id": 2, + "interval": "$interval", + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceph_osd_stat_bytes_used{ceph_daemon=~\"$osd\",application=\"ceph\",release_group=\"$ceph_cluster\"}", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Used - {{ osd.$osd }}", + "metric": "ceph_osd_used_bytes", + "refId": "A", + "step": 60 + }, + { + "expr": "ceph_osd_stat_bytes{ceph_daemon=~\"$osd\",application=\"ceph\",release_group=\"$ceph_cluster\"} - ceph_osd_stat_bytes_used{ceph_daemon=~\"$osd\",application=\"ceph\",release_group=\"$ceph_cluster\"}", + "hide": false, + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Available - {{ $osd }}", + "metric": "ceph_osd_avail_bytes", + "refId": "B", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "OSD Storage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 5, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 13 + }, + "id": 9, + "interval": "$interval", + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 2, + "points": true, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "(ceph_osd_stat_bytes_used{ceph_daemon=~\"$osd\",application=\"ceph\",release_group=\"$ceph_cluster\"}/ceph_osd_stat_bytes{ceph_daemon=~\"$osd\",application=\"ceph\",release_group=\"$ceph_cluster\"})", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Available - {{ $osd }}", + "metric": "ceph_osd_avail_bytes", + "refId": "A", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Utilization Variance", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "15m", + "schemaVersion": 18, + "style": "dark", + "tags": [ + "ceph", + "osd" + ], + "templating": { + "list": [ + { + "current": { + "text": "prometheus", + "value": "prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Prometheus datasource", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "clcp-ucp-ceph-client", + "value": "clcp-ucp-ceph-client" + }, + "datasource": "${DS_PROMETHEUS}", + "definition": "", + "hide": 0, + "includeAll": false, + "label": "Cluster", + "multi": false, + "name": "ceph_cluster", + "options": [], + "query": "label_values(ceph_health_status, release_group)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "auto": true, + "auto_count": 10, + "auto_min": "1m", + "current": { + "text": "1m", + "value": "1m" + }, + "datasource": null, + "hide": 0, + "includeAll": false, + "label": "Interval", + "multi": false, + "name": "interval", + "options": [ + { + "selected": false, + "text": "auto", + "value": "$__auto_interval_interval" + }, + { + "selected": true, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + }, + { + "allValue": null, + "current": { + "text": "osd.0", + "value": "osd.0" + }, + "datasource": "${DS_PROMETHEUS}", + "definition": "", + "hide": 0, + "includeAll": false, + "label": "OSD", + "multi": false, + "name": "osd", + "options": [], + "query": "label_values(ceph_osd_metadata{release_group=\"$ceph_cluster\"}, ceph_daemon)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Ceph - OSD", + "version": 1 + } + ceph_pool: |- + { + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "prometheus", + "description": "Prometheus.IO", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "3.1.1" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "singlestat", + "name": "Singlestat", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Ceph Pools dashboard.", + "overwrite": true, + "editable": false, + "gnetId": 926, + "graphTooltip": 0, + "id": 2, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 11, + "panels": [], + "title": "Pool: $pool", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "editable": true, + "error": false, + "fill": 4, + "grid": {}, + "gridPos": { + "h": 7, + "w": 20, + "x": 0, + "y": 1 + }, + "height": "", + "id": 2, + "interval": "$interval", + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 0, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/^Total.*$/", + "fill": 0, + "linewidth": 4, + "stack": false + }, + { + "alias": "/^Raw.*$/", + "color": "#BF1B00", + "fill": 0, + "linewidth": 4 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "ceph_pool_max_avail{pool_id=~\"$pool\",application=\"ceph\",release_group=\"$ceph_cluster\"}", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Total - {{ $pool }}", + "refId": "A", + "step": 60 + }, + { + "expr": "ceph_pool_stored{pool_id=~\"$pool\",application=\"ceph\",release_group=\"$ceph_cluster\"}", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Used - {{ $pool }}", + "refId": "B", + "step": 60 + }, + { + "expr": "ceph_pool_max_avail{pool_id=~\"$pool\",application=\"ceph\",release_group=\"$ceph_cluster\"} - ceph_pool_stored{pool_id=~\"$pool\",application=\"ceph\",release_group=\"$ceph_cluster\"}", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Available - {{ $pool }}", + "refId": "C", + "step": 60 + }, + { + "expr": "ceph_pool_raw_bytes_used{pool_id=~\"$pool\",application=\"ceph\",release_group=\"$ceph_cluster\"}", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Raw - {{ $pool }}", + "refId": "D", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "[[pool_name]] Pool Storage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "editable": true, + "error": false, + "format": "percentunit", + "gauge": { + "maxValue": 1, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 20, + "y": 1 + }, + "id": 10, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(ceph_pool_stored{pool_id=~\"$pool\",application=\"ceph\",release_group=\"$ceph_cluster\"} / ceph_pool_max_avail{pool_id=~\"$pool\",application=\"ceph\",release_group=\"$ceph_cluster\"})", + "format": "time_series", + "interval": "$interval", + "intervalFactor": 1, + "refId": "A", + "step": 60 + } + ], + "thresholds": "", + "title": "[[pool_name]] Pool Usage", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 12, + "panels": [], + "title": "New row", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 9 + }, + "height": "", + "id": 7, + "isNew": true, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "ceph_pool_objects{pool_id=~\"$pool\",application=\"ceph\",release_group=\"$ceph_cluster\"}", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Objects - {{ $pool_name }}", + "refId": "A", + "step": 60 + }, + { + "expr": "ceph_pool_dirty{pool_id=~\"$pool\",application=\"ceph\",release_group=\"$ceph_cluster\"}", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Dirty Objects - {{ $pool_name }}", + "refId": "B", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Objects in Pool [[pool_name]]", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 9 + }, + "id": 4, + "interval": "$interval", + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "irate(ceph_pool_rd{pool_id=~\"$pool\",application=\"ceph\",release_group=\"$ceph_cluster\"}[3m])", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Read - {{ $pool_name }}", + "refId": "B", + "step": 60 + }, + { + "expr": "irate(ceph_pool_wr{pool_id=~\"$pool\",application=\"ceph\",release_group=\"$ceph_cluster\"}[3m])", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Write - {{ $pool_name }}", + "refId": "A", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "[[pool_name]] Pool IOPS", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": "IOPS", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": "IOPS", + "logBase": 1, + "max": null, + "min": 0, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 5, + "interval": "$interval", + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "irate(ceph_pool_rd_bytes{pool_id=\"$pool\",application=\"ceph\",release_group=\"$ceph_cluster\"}[3m])", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Read Bytes - {{ $pool_name }}", + "refId": "A", + "step": 60 + }, + { + "expr": "irate(ceph_pool_wr_bytes{pool_id=\"$pool\",application=\"ceph\",release_group=\"$ceph_cluster\"}[3m])", + "interval": "$interval", + "intervalFactor": 1, + "legendFormat": "Written Bytes - {{ $pool_name }}", + "refId": "B", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "[[pool_name]] Pool Throughput", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "5m", + "schemaVersion": 18, + "style": "dark", + "tags": [ + "ceph", + "pools" + ], + "templating": { + "list": [ + { + "current": { + "text": "prometheus", + "value": "prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Prometheus datasource", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": null, + "current": { + "text": "clcp-ucp-ceph-client", + "value": "clcp-ucp-ceph-client" + }, + "datasource": "${DS_PROMETHEUS}", + "definition": "", + "hide": 0, + "includeAll": false, + "label": "Cluster", + "multi": false, + "name": "ceph_cluster", + "options": [], + "query": "label_values(ceph_health_status, release_group)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "auto": true, + "auto_count": 10, + "auto_min": "1m", + "current": { + "text": "1m", + "value": "1m" + }, + "datasource": null, + "hide": 0, + "includeAll": false, + "label": "Interval", + "multi": false, + "name": "interval", + "options": [ + { + "selected": false, + "text": "auto", + "value": "$__auto_interval_interval" + }, + { + "selected": true, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + }, + { + "allValue": null, + "current": { + "text": "1", + "value": "1" + }, + "datasource": "${DS_PROMETHEUS}", + "definition": "", + "hide": 0, + "includeAll": false, + "label": "Pool", + "multi": false, + "name": "pool", + "options": [], + "query": "label_values(ceph_pool_objects{release_group=\"$ceph_cluster\"}, pool_id)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "rbd", + "value": "rbd" + }, + "datasource": "${DS_PROMETHEUS}", + "definition": "", + "hide": 0, + "includeAll": false, + "label": "Pool", + "multi": false, + "name": "pool_name", + "options": [], + "query": "label_values(ceph_pool_metadata{release_group=\"$ceph_cluster\",pool_id=\"[[pool]]\" }, name)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Ceph - Pools", + "version": 1 + } +... diff --git a/values_overrides/grafana/containers.yaml b/values_overrides/grafana/containers.yaml new file mode 100644 index 0000000000..67e9217a8d --- /dev/null +++ b/values_overrides/grafana/containers.yaml @@ -0,0 +1,2106 @@ +# NOTE(srwilkers): This overrides file provides a reference for a dashboard for +# container metrics, specific to each host +--- +conf: + dashboards: + kubernetes: + containers: |- + { + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "3.1.1" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.3.0" + }, + { + "type": "panel", + "id": "singlestat", + "name": "Singlestat", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Monitors Kubernetes cluster using Prometheus. Shows overall cluster CPU / Memory / Filesystem usage as well as individual pod, containers, systemd services statistics. Uses cAdvisor metrics only.", + "overwrite": true, + "editable": false, + "gnetId": 315, + "graphTooltip": 0, + "id": 32, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 33, + "panels": [], + "title": "Network I/O pressure", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 5, + "w": 24, + "x": 0, + "y": 1 + }, + "height": "200px", + "id": 32, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (rate (container_network_receive_bytes_total{kubernetes_io_hostname=~\"^$Node$\"}[5m]))", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "Received", + "metric": "network", + "refId": "A", + "step": 10 + }, + { + "expr": "- sum (rate (container_network_transmit_bytes_total{kubernetes_io_hostname=~\"^$Node$\"}[5m]))", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "Sent", + "metric": "network", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network I/O pressure", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 34, + "panels": [], + "title": "Total usage", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 7 + }, + "height": "180px", + "id": 4, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum (container_memory_working_set_bytes{id=\"/\",kubernetes_io_hostname=~\"^$Node$\"}) / sum (machine_memory_bytes{kubernetes_io_hostname=~\"^$Node$\"}) * 100", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "65, 90", + "title": "Cluster memory usage", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 7 + }, + "height": "180px", + "id": 6, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum (rate (container_cpu_usage_seconds_total{id=\"/\",kubernetes_io_hostname=~\"^$Node$\"}[5m])) / sum (machine_cpu_cores{kubernetes_io_hostname=~\"^$Node$\"}) * 100", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "65, 90", + "title": "Cluster CPU usage (5m avg)", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 16, + "y": 7 + }, + "height": "180px", + "id": 7, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum (container_fs_usage_bytes{device=~\"^/dev/[sv]da[0-9]$\",id=~\"/.+\",kubernetes_io_hostname=~\"^$Node$\"}) / sum (container_fs_limit_bytes{device=~\"^/dev/[sv]da[0-9]$\",id=~\"/.+\",kubernetes_io_hostname=~\"^$Node$\"}) * 100", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 10 + } + ], + "thresholds": "65, 90", + "title": "Cluster filesystem usage", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "editable": true, + "error": false, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 12 + }, + "height": "1px", + "id": 9, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "20%", + "prefix": "", + "prefixFontSize": "20%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum (container_memory_working_set_bytes{id=\"/\",kubernetes_io_hostname=~\"^$Node$\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Used", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "editable": true, + "error": false, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 12 + }, + "height": "1px", + "id": 10, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum (machine_memory_bytes{kubernetes_io_hostname=~\"^$Node$\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Total", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 12 + }, + "height": "1px", + "id": 11, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": " cores", + "postfixFontSize": "30%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum (rate (container_cpu_usage_seconds_total{id=\"/\",kubernetes_io_hostname=~\"^$Node$\"}[5m]))", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Used", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 12 + }, + "height": "1px", + "id": 12, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": " cores", + "postfixFontSize": "30%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum (machine_cpu_cores{kubernetes_io_hostname=~\"^$Node$\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Total", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "editable": true, + "error": false, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 12 + }, + "height": "1px", + "id": 13, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum (container_fs_usage_bytes{device=~\"^/dev/[sv]da[0-9]$\",id=~\"/.+\",kubernetes_io_hostname=~\"^$Node$\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Used", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "editable": true, + "error": false, + "format": "bytes", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 12 + }, + "height": "1px", + "id": 14, + "interval": null, + "isNew": true, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum (container_fs_limit_bytes{device=~\"^/dev/[sv]da[0-9]$\",id=~\"/.+\",kubernetes_io_hostname=~\"^$Node$\"})", + "interval": "10s", + "intervalFactor": 1, + "refId": "A", + "step": 10 + } + ], + "thresholds": "", + "title": "Total", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 35, + "panels": [], + "title": "Pods CPU usage", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 3, + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 16 + }, + "height": "", + "id": 17, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": true, + "targets": [ + { + "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",kubernetes_io_hostname=~\"^$Node$\"}[5m])) by (pod)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "{{ pod }}", + "metric": "container_cpu", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Pods CPU usage (5m avg)", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": "cores", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 23 + }, + "id": 36, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 3, + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 23 + }, + "height": "", + "id": 24, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": null, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "stack": false, + "steppedLine": true, + "targets": [ + { + "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",name=~\"^k8s_.*\",container!=\"POD\",kubernetes_io_hostname=~\"^$Node$\"}[5m])) by (container, pod)", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "pod: {{ pod }} | {{ container }}", + "metric": "container_cpu", + "refId": "A", + "step": 10 + }, + { + "expr": "sum (rate (container_cpu_usage_seconds_total{image!=\"\",name!~\"^k8s_.*\",kubernetes_io_hostname=~\"^$Node$\"}[5m])) by (kubernetes_io_hostname, name, image)", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "docker: {{ kubernetes_io_hostname }} | {{ image }} ({{ name }})", + "metric": "container_cpu", + "refId": "B", + "step": 10 + }, + { + "expr": "sum (rate (container_cpu_usage_seconds_total{rkt_container_name!=\"\",kubernetes_io_hostname=~\"^$Node$\"}[5m])) by (kubernetes_io_hostname, rkt_container_name)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "rkt: {{ kubernetes_io_hostname }} | {{ rkt_container_name }}", + "metric": "container_cpu", + "refId": "C", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Containers CPU usage (5m avg)", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "none", + "label": "cores", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "title": "Containers CPU usage", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 37, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 3, + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 13, + "w": 24, + "x": 0, + "y": 24 + }, + "id": 20, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "stack": false, + "steppedLine": true, + "targets": [ + { + "expr": "sum (rate (container_cpu_usage_seconds_total{id!=\"/\",kubernetes_io_hostname=~\"^$Node$\"}[5m])) by (id)", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "{{ id }}", + "metric": "container_cpu", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "All processes CPU usage (5m avg)", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "none", + "label": "cores", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "repeat": null, + "title": "All processes CPU usage", + "type": "row" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 38, + "panels": [], + "title": "Pods memory usage", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 26 + }, + "id": 25, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": true, + "targets": [ + { + "expr": "sum (container_memory_working_set_bytes{image!=\"\",kubernetes_io_hostname=~\"^$Node$\"}) by (pod)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "{{ pod }}", + "metric": "container_memory_usage:sort_desc", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Pods memory usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 39, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 33 + }, + "id": 27, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "stack": false, + "steppedLine": true, + "targets": [ + { + "expr": "sum (container_memory_working_set_bytes{image!=\"\",name=~\"^k8s_.*\",container!=\"POD\",kubernetes_io_hostname=~\"^$Node$\"}) by (container, pod)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "pod: {{ pod }} | {{ container }}", + "metric": "container_memory_usage:sort_desc", + "refId": "A", + "step": 10 + }, + { + "expr": "sum (container_memory_working_set_bytes{image!=\"\",name!~\"^k8s_.*\",kubernetes_io_hostname=~\"^$Node$\"}) by (kubernetes_io_hostname, name, image)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "docker: {{ kubernetes_io_hostname }} | {{ image }} ({{ name }})", + "metric": "container_memory_usage:sort_desc", + "refId": "B", + "step": 10 + }, + { + "expr": "sum (container_memory_working_set_bytes{rkt_container_name!=\"\",kubernetes_io_hostname=~\"^$Node$\"}) by (kubernetes_io_hostname, rkt_container_name)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "rkt: {{ kubernetes_io_hostname }} | {{ rkt_container_name }}", + "metric": "container_memory_usage:sort_desc", + "refId": "C", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Containers memory usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "title": "Containers memory usage", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 40, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 13, + "w": 24, + "x": 0, + "y": 34 + }, + "id": 28, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "stack": false, + "steppedLine": true, + "targets": [ + { + "expr": "sum (container_memory_working_set_bytes{id!=\"/\",kubernetes_io_hostname=~\"^$Node$\"}) by (id)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "{{ id }}", + "metric": "container_memory_usage:sort_desc", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "All processes memory usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "title": "All processes memory usage", + "type": "row" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 35 + }, + "id": 41, + "panels": [], + "title": "Pods network I/O", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 16, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (rate (container_network_receive_bytes_total{image!=\"\",kubernetes_io_hostname=~\"^$Node$\"}[5m])) by (pod)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "-> {{ pod }}", + "metric": "network", + "refId": "A", + "step": 10 + }, + { + "expr": "- sum (rate (container_network_transmit_bytes_total{image!=\"\",kubernetes_io_hostname=~\"^$Node$\"}[5m])) by (pod)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "<- {{ pod }}", + "metric": "network", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Pods network I/O (5m avg)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 43 + }, + "id": 42, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 43 + }, + "id": 30, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (rate (container_network_receive_bytes_total{image!=\"\",name=~\"^k8s_.*\",kubernetes_io_hostname=~\"^$Node$\"}[5m])) by (container, pod)", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "-> pod: {{ pod }} | {{ container }}", + "metric": "network", + "refId": "B", + "step": 10 + }, + { + "expr": "- sum (rate (container_network_transmit_bytes_total{image!=\"\",name=~\"^k8s_.*\",kubernetes_io_hostname=~\"^$Node$\"}[5m])) by (container, pod)", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "<- pod: {{ pod }} | {{ container }}", + "metric": "network", + "refId": "D", + "step": 10 + }, + { + "expr": "sum (rate (container_network_receive_bytes_total{image!=\"\",name!~\"^k8s_.*\",kubernetes_io_hostname=~\"^$Node$\"}[5m])) by (kubernetes_io_hostname, name, image)", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "-> docker: {{ kubernetes_io_hostname }} | {{ image }} ({{ name }})", + "metric": "network", + "refId": "A", + "step": 10 + }, + { + "expr": "- sum (rate (container_network_transmit_bytes_total{image!=\"\",name!~\"^k8s_.*\",kubernetes_io_hostname=~\"^$Node$\"}[5m])) by (kubernetes_io_hostname, name, image)", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "<- docker: {{ kubernetes_io_hostname }} | {{ image }} ({{ name }})", + "metric": "network", + "refId": "C", + "step": 10 + }, + { + "expr": "sum (rate (container_network_transmit_bytes_total{rkt_container_name!=\"\",kubernetes_io_hostname=~\"^$Node$\"}[5m])) by (kubernetes_io_hostname, rkt_container_name)", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "-> rkt: {{ kubernetes_io_hostname }} | {{ rkt_container_name }}", + "metric": "network", + "refId": "E", + "step": 10 + }, + { + "expr": "- sum (rate (container_network_transmit_bytes_total{rkt_container_name!=\"\",kubernetes_io_hostname=~\"^$Node$\"}[5m])) by (kubernetes_io_hostname, rkt_container_name)", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "<- rkt: {{ kubernetes_io_hostname }} | {{ rkt_container_name }}", + "metric": "network", + "refId": "F", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Containers network I/O (5m avg)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "title": "Containers network I/O", + "type": "row" + }, + { + "collapsed": true, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 44 + }, + "id": 43, + "panels": [ + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 13, + "w": 24, + "x": 0, + "y": 44 + }, + "id": 29, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (rate (container_network_receive_bytes_total{id!=\"/\",kubernetes_io_hostname=~\"^$Node$\"}[5m])) by (id)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "-> {{ id }}", + "metric": "network", + "refId": "A", + "step": 10 + }, + { + "expr": "- sum (rate (container_network_transmit_bytes_total{id!=\"/\",kubernetes_io_hostname=~\"^$Node$\"}[5m])) by (id)", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "<- {{ id }}", + "metric": "network", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "All processes network I/O (5m avg)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "show": true + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "title": "All processes network I/O", + "type": "row" + } + ], + "refresh": "5m", + "schemaVersion": 18, + "style": "dark", + "tags": [ + "kubernetes" + ], + "templating": { + "list": [ + { + "current": { + "text": "prometheus", + "value": "prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Prometheus datasource", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "${DS_PROMETHEUS}", + "definition": "", + "hide": 0, + "includeAll": true, + "label": null, + "multi": false, + "name": "Node", + "options": [], + "query": "label_values(kubernetes_io_hostname)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-5m", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Container Metrics (cAdvisor)", + "version": 1 + } +... diff --git a/values_overrides/grafana/coredns.yaml b/values_overrides/grafana/coredns.yaml new file mode 100644 index 0000000000..d26f800065 --- /dev/null +++ b/values_overrides/grafana/coredns.yaml @@ -0,0 +1,1382 @@ +# NOTE(srwilkers): This overrides file provides a reference for a dashboard for +# CoreDNS +--- +conf: + dashboards: + kubernetes: + coredns: |- + { + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "4.4.3" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "A dashboard for the CoreDNS DNS server.", + "overwrite": true, + "editable": true, + "gnetId": 5926, + "graphTooltip": 0, + "id": 20, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 0 + }, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_count_total{instance=~\"$instance\"}[5m])) by (proto)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{proto}}", + "refId": "A", + "step": 60 + }, + { + "expr": "sum(rate(coredns_dns_request_count_total{instance=~\"$instance\"}[5m]))", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "total", + "refId": "B", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (total)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 0 + }, + "id": 12, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "yaxis": 2 + }, + { + "alias": "other", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_type_count_total{instance=~\"$instance\"}[5m])) by (type)", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "A", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (by qtype)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 0 + }, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_count_total{instance=~\"$instance\"}[5m])) by (zone)", + "intervalFactor": 2, + "legendFormat": "{{zone}}", + "refId": "A", + "step": 60 + }, + { + "expr": "sum(rate(coredns_dns_request_count_total{instance=~\"$instance\"}[5m]))", + "intervalFactor": 2, + "legendFormat": "total", + "refId": "B", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (by zone)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "total", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_request_do_count_total{instance=~\"$instance\"}[5m]))", + "intervalFactor": 2, + "legendFormat": "DO", + "refId": "A", + "step": 40 + }, + { + "expr": "sum(rate(coredns_dns_request_count_total{instance=~\"$instance\"}[5m]))", + "intervalFactor": 2, + "legendFormat": "total", + "refId": "B", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (DO bit)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 7 + }, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "tcp:90", + "yaxis": 2 + }, + { + "alias": "tcp:99 ", + "yaxis": 2 + }, + { + "alias": "tcp:50", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_request_size_bytes_bucket{instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto))", + "intervalFactor": 2, + "legendFormat": "{{proto}}:99 ", + "refId": "A", + "step": 60 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_request_size_bytes_bucket{instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto))", + "intervalFactor": 2, + "legendFormat": "{{proto}}:90", + "refId": "B", + "step": 60 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_request_size_bytes_bucket{instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto))", + "intervalFactor": 2, + "legendFormat": "{{proto}}:50", + "refId": "C", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (size, udp)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 7 + }, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "tcp:90", + "yaxis": 1 + }, + { + "alias": "tcp:99 ", + "yaxis": 1 + }, + { + "alias": "tcp:50", + "yaxis": 1 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_request_size_bytes_bucket{instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto))", + "intervalFactor": 2, + "legendFormat": "{{proto}}:99 ", + "refId": "A", + "step": 60 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_request_size_bytes_bucket{instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto))", + "intervalFactor": 2, + "legendFormat": "{{proto}}:90", + "refId": "B", + "step": 60 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_request_size_bytes_bucket{instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto))", + "intervalFactor": 2, + "legendFormat": "{{proto}}:50", + "refId": "C", + "step": 60 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Requests (size,tcp)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 14 + }, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_dns_response_rcode_count_total{instance=~\"$instance\"}[5m])) by (rcode)", + "intervalFactor": 2, + "legendFormat": "{{rcode}}", + "refId": "A", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Responses (by rcode)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 14 + }, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_request_duration_seconds_bucket{instance=~\"$instance\"}[5m])) by (le, job))", + "intervalFactor": 2, + "legendFormat": "99%", + "refId": "A", + "step": 40 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_request_duration_seconds_bucket{instance=~\"$instance\"}[5m])) by (le))", + "intervalFactor": 2, + "legendFormat": "90%", + "refId": "B", + "step": 40 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_request_duration_seconds_bucket{instance=~\"$instance\"}[5m])) by (le))", + "intervalFactor": 2, + "legendFormat": "50%", + "refId": "C", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Responses (duration)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "udp:50%", + "yaxis": 1 + }, + { + "alias": "tcp:50%", + "yaxis": 2 + }, + { + "alias": "tcp:90%", + "yaxis": 2 + }, + { + "alias": "tcp:99%", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_response_size_bytes_bucket{instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto)) ", + "intervalFactor": 2, + "legendFormat": "{{proto}}:99%", + "refId": "A", + "step": 40 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_response_size_bytes_bucket{instance=\"$instance\",proto=\"udp\"}[5m])) by (le,proto)) ", + "intervalFactor": 2, + "legendFormat": "{{proto}}:90%", + "refId": "B", + "step": 40 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_response_size_bytes_bucket{instance=~\"$instance\",proto=\"udp\"}[5m])) by (le,proto)) ", + "intervalFactor": 2, + "legendFormat": "{{proto}}:50%", + "metric": "", + "refId": "C", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Responses (size, udp)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "udp:50%", + "yaxis": 1 + }, + { + "alias": "tcp:50%", + "yaxis": 1 + }, + { + "alias": "tcp:90%", + "yaxis": 1 + }, + { + "alias": "tcp:99%", + "yaxis": 1 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(coredns_dns_response_size_bytes_bucket{instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto)) ", + "intervalFactor": 2, + "legendFormat": "{{proto}}:99%", + "refId": "A", + "step": 40 + }, + { + "expr": "histogram_quantile(0.90, sum(rate(coredns_dns_response_size_bytes_bucket{instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le,proto)) ", + "intervalFactor": 2, + "legendFormat": "{{proto}}:90%", + "refId": "B", + "step": 40 + }, + { + "expr": "histogram_quantile(0.50, sum(rate(coredns_dns_response_size_bytes_bucket{instance=~\"$instance\",proto=\"tcp\"}[5m])) by (le, proto)) ", + "intervalFactor": 2, + "legendFormat": "{{proto}}:50%", + "metric": "", + "refId": "C", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Responses (size, tcp)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 28 + }, + "id": 15, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(coredns_cache_size{instance=~\"$instance\"}) by (type)", + "intervalFactor": 2, + "legendFormat": "{{type}}", + "refId": "A", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cache (size)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 28 + }, + "id": 16, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "misses", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(coredns_cache_hits_total{instance=~\"$instance\"}[5m])) by (type)", + "intervalFactor": 2, + "legendFormat": "hits:{{type}}", + "refId": "A", + "step": 40 + }, + { + "expr": "sum(rate(coredns_cache_misses_total{instance=~\"$instance\"}[5m])) by (type)", + "intervalFactor": 2, + "legendFormat": "misses", + "refId": "B", + "step": 40 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cache (hitrate)", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "pps", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "schemaVersion": 18, + "style": "dark", + "tags": [ + "dns", + "coredns" + ], + "templating": { + "list": [ + { + "current": { + "text": "prometheus", + "value": "prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Prometheus datasource", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "${DS_PROMETHEUS}", + "definition": "", + "hide": 0, + "includeAll": true, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [], + "query": "up{job=\"coredns\"}", + "refresh": 1, + "regex": ".*instance=\"(.*?)\".*", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "CoreDNS", + "version": 1 + } +... diff --git a/values_overrides/grafana/elasticsearch.yaml b/values_overrides/grafana/elasticsearch.yaml new file mode 100644 index 0000000000..5836da759b --- /dev/null +++ b/values_overrides/grafana/elasticsearch.yaml @@ -0,0 +1,3478 @@ +# NOTE(srwilkers): This overrides file provides a reference for a dashboard for +# an Elasticsearch cluster +--- +conf: + dashboards: + lma: + elasticsearch: |- + { + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "4.6.3" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "singlestat", + "name": "Singlestat", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Elasticsearch detailed dashboard", + "overwrite": true, + "editable": true, + "gnetId": 4358, + "graphTooltip": 1, + "id": 23, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 50, + "panels": [], + "repeat": null, + "title": "Cluster", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(178, 49, 13, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 10, + "x": 0, + "y": 1 + }, + "height": "50", + "id": 8, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "(sum(elasticsearch_cluster_health_status{cluster=~\"$cluster\",color=\"green\"})*2)+sum(elasticsearch_cluster_health_status{cluster=~\"$cluster\",color=\"yellow\"})", + "format": "time_series", + "intervalFactor": 3, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 40 + } + ], + "thresholds": "0,1,2", + "title": "Cluster health status", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "GREEN", + "value": "2" + }, + { + "op": "=", + "text": "YELLOW", + "value": "1" + }, + { + "op": "=", + "text": "RED", + "value": "0" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 10, + "y": 1 + }, + "height": "50", + "id": 10, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(elasticsearch_cluster_health_number_of_nodes{cluster=~\"$cluster\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 40 + } + ], + "thresholds": "", + "title": "Nodes", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 14, + "y": 1 + }, + "height": "50", + "id": 9, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "elasticsearch_cluster_health_number_of_data_nodes{cluster=\"$cluster\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 40 + } + ], + "thresholds": "", + "title": "Data nodes", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 18, + "y": 1 + }, + "height": "50", + "hideTimeOverride": true, + "id": 16, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "elasticsearch_cluster_health_number_of_pending_tasks{cluster=\"$cluster\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "metric": "", + "refId": "A", + "step": 40 + } + ], + "thresholds": "", + "title": "Pending tasks", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 4 + }, + "id": 51, + "panels": [], + "repeat": null, + "title": "Shards", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 0, + "y": 5 + }, + "height": "50", + "id": 11, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "maxPerRow": 6, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "repeat": "shard_type", + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "elasticsearch_cluster_health_active_primary_shards{cluster=\"$cluster\"}", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 40 + } + ], + "thresholds": "", + "title": "active primary shards", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 4, + "y": 5 + }, + "height": "50", + "id": 39, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "maxPerRow": 6, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "elasticsearch_cluster_health_active_shards{cluster=\"$cluster\"}", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 40 + } + ], + "thresholds": "", + "title": "active shards", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 8, + "y": 5 + }, + "height": "50", + "id": 40, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "maxPerRow": 6, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "elasticsearch_cluster_health_initializing_shards{cluster=\"$cluster\"}", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 40 + } + ], + "thresholds": "", + "title": "initializing shards", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 12, + "y": 5 + }, + "height": "50", + "id": 41, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "maxPerRow": 6, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "elasticsearch_cluster_health_relocating_shards{cluster=\"$cluster\"}", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 40 + } + ], + "thresholds": "", + "title": "relocating shards", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 5 + }, + "height": "50", + "id": 42, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "maxPerRow": 6, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "elasticsearch_cluster_health_unassigned_shards{cluster=\"$cluster\"}", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 40 + } + ], + "thresholds": "", + "title": "unassigned shards", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 8 + }, + "id": 52, + "panels": [], + "repeat": null, + "title": "System", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 10, + "w": 6, + "x": 0, + "y": 9 + }, + "height": "400", + "id": 30, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "elasticsearch_process_cpu_percent{cluster=\"$cluster\",es_master_node=\"true\",name=~\"$node\"}", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ name }} - master", + "metric": "", + "refId": "A", + "step": 10 + }, + { + "expr": "elasticsearch_process_cpu_percent{cluster=\"$cluster\",es_data_node=\"true\",name=~\"$node\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ name }} - data", + "metric": "", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "CPU usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": "CPU usage", + "logBase": 1, + "max": 100, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 10, + "w": 6, + "x": 6, + "y": 9 + }, + "height": "400", + "id": 31, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "elasticsearch_jvm_memory_used_bytes{cluster=\"$cluster\",name=~\"$node\",name=~\"$node\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ name }} - used: {{area}}", + "metric": "", + "refId": "A", + "step": 10 + }, + { + "expr": "elasticsearch_jvm_memory_committed_bytes{cluster=\"$cluster\",name=~\"$node\",name=~\"$node\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ name }} - committed: {{area}}", + "refId": "B", + "step": 10 + }, + { + "expr": "elasticsearch_jvm_memory_max_bytes{cluster=\"$cluster\",name=~\"$node\",name=~\"$node\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ name }} - max: {{area}}", + "refId": "C", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "JVM memory usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "Memory", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 10, + "w": 6, + "x": 12, + "y": 9 + }, + "height": "400", + "id": 32, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "1-(elasticsearch_filesystem_data_available_bytes{cluster=\"$cluster\"}/elasticsearch_filesystem_data_size_bytes{cluster=\"$cluster\",name=~\"$node\"})", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ name }} - {{path}}", + "metric": "", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + { + "colorMode": "custom", + "fill": true, + "fillColor": "rgba(216, 200, 27, 0.27)", + "op": "gt", + "value": 0.8 + }, + { + "colorMode": "custom", + "fill": true, + "fillColor": "rgba(234, 112, 112, 0.22)", + "op": "gt", + "value": 0.9 + } + ], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": "Disk Usage %", + "logBase": 1, + "max": 1, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 10, + "w": 6, + "x": 18, + "y": 9 + }, + "height": "400", + "id": 47, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "sent", + "transform": "negative-Y" + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(elasticsearch_transport_tx_size_bytes_total{cluster=\"$cluster\",name=~\"$node\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ name }} -sent", + "refId": "D", + "step": 10 + }, + { + "expr": "irate(elasticsearch_transport_rx_size_bytes_total{cluster=\"$cluster\",name=~\"$node\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ name }} -received", + "refId": "C", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": "Bytes/sec", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "pps", + "label": "", + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 53, + "panels": [], + "repeat": null, + "title": "Documents", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 10, + "w": 6, + "x": 0, + "y": 20 + }, + "height": "400", + "id": 1, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "elasticsearch_indices_docs{cluster=\"$cluster\",name=~\"$node\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ name }}", + "metric": "", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Documents count", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Documents", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 10, + "w": 6, + "x": 6, + "y": 20 + }, + "height": "400", + "id": 24, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "irate(elasticsearch_indices_indexing_index_total{cluster=\"$cluster\",name=~\"$node\"}[$interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Documents indexed rate", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "index calls/s", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 10, + "w": 6, + "x": 12, + "y": 20 + }, + "height": "400", + "id": 25, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(elasticsearch_indices_docs_deleted{cluster=\"$cluster\",name=~\"$node\"}[$interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Documents deleted rate", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Documents/s", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 10, + "w": 6, + "x": 18, + "y": 20 + }, + "height": "400", + "id": 26, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(elasticsearch_indices_merges_total{cluster=\"$cluster\",name=~\"$node\"}[$interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Documents merged rate", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Documents/s", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 54, + "panels": [], + "repeat": null, + "title": "Total Operations stats", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 31 + }, + "height": "400", + "id": 48, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(elasticsearch_indices_indexing_index_total{cluster=\"$cluster\",name=~\"$node\"}[$interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ name }} - indexing", + "metric": "", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(elasticsearch_indices_search_query_total{cluster=\"$cluster\",name=~\"$node\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ name }} - query", + "refId": "B", + "step": 4 + }, + { + "expr": "irate(elasticsearch_indices_search_fetch_total{cluster=\"$cluster\",name=~\"$node\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ name }} - fetch", + "refId": "C", + "step": 4 + }, + { + "expr": "irate(elasticsearch_indices_merges_total{cluster=\"$cluster\",name=~\"$node\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ name }} - merges", + "refId": "D", + "step": 4 + }, + { + "expr": "irate(elasticsearch_indices_refresh_total{cluster=\"$cluster\",name=~\"$node\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ name }} - refresh", + "refId": "E", + "step": 4 + }, + { + "expr": "irate(elasticsearch_indices_flush_total{cluster=\"$cluster\",name=~\"$node\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ name }} - flush", + "refId": "F", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total Operations rate", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Operations/s", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 31 + }, + "height": "400", + "id": 49, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(elasticsearch_indices_indexing_index_time_seconds_total{cluster=\"$cluster\",name=~\"$node\"}[$interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{ name }} - indexing", + "metric": "", + "refId": "A", + "step": 4 + }, + { + "expr": "irate(elasticsearch_indices_search_query_time_ms_total{cluster=\"$cluster\",name=~\"$node\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ name }} - query", + "refId": "B", + "step": 4 + }, + { + "expr": "irate(elasticsearch_indices_search_fetch_time_ms_total{cluster=\"$cluster\",name=~\"$node\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ name }} - fetch", + "refId": "C", + "step": 4 + }, + { + "expr": "irate(elasticsearch_indices_merges_total_time_ms_total{cluster=\"$cluster\",name=~\"$node\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ name }} - merges", + "refId": "D", + "step": 4 + }, + { + "expr": "irate(elasticsearch_indices_refresh_total_time_ms_total{cluster=\"$cluster\",name=~\"$node\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ name }} - refresh", + "refId": "E", + "step": 4 + }, + { + "expr": "irate(elasticsearch_indices_flush_time_ms_total{cluster=\"$cluster\",name=~\"$node\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ name }} - flush", + "refId": "F", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Total Operations time", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": "Time", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 41 + }, + "id": 55, + "panels": [], + "repeat": null, + "title": "Times", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 10, + "w": 8, + "x": 0, + "y": 42 + }, + "height": "400", + "id": 33, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(elasticsearch_indices_search_query_time_seconds{cluster=\"$cluster\",name=~\"$node\"}[$interval]) ", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Query time", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": "Time", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 10, + "w": 8, + "x": 8, + "y": 42 + }, + "height": "400", + "id": 5, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(elasticsearch_indices_indexing_index_time_seconds_total{cluster=\"$cluster\",name=~\"$node\"}[$interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Indexing time", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": "Time", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 42 + }, + "height": "400", + "id": 3, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(elasticsearch_indices_merges_total_time_seconds_total{cluster=\"$cluster\",name=~\"$node\"}[$interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Merging time", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": "Time", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 52 + }, + "id": 56, + "panels": [], + "repeat": null, + "title": "Caches", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 10, + "w": 6, + "x": 0, + "y": 53 + }, + "height": "400", + "id": 4, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "elasticsearch_indices_fielddata_memory_size_bytes{cluster=\"$cluster\",name=~\"$node\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Field data memory size", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "Memory", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 10, + "w": 6, + "x": 6, + "y": 53 + }, + "height": "400", + "id": 34, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(elasticsearch_indices_fielddata_evictions{cluster=\"$cluster\",name=~\"$node\"}[$interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Field data evictions", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Evictions/s", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 10, + "w": 6, + "x": 12, + "y": 53 + }, + "height": "400", + "id": 35, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "elasticsearch_indices_query_cache_memory_size_bytes{cluster=\"$cluster\",name=~\"$node\"}", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Query cache size", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": "Size", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 10, + "w": 6, + "x": 18, + "y": 53 + }, + "height": "400", + "id": 36, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(elasticsearch_indices_query_cache_evictions{cluster=\"$cluster\",name=~\"$node\"}[$interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}}", + "metric": "", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Query cache evictions", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "Evictions/s", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 63 + }, + "id": 57, + "panels": [], + "repeat": null, + "title": "Thread Pool", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 19, + "w": 6, + "x": 0, + "y": 64 + }, + "id": 45, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": " irate(elasticsearch_thread_pool_rejected_count{cluster=\"$cluster\",name=~\"$node\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}} - {{ type }}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Thread Pool operations rejected", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 19, + "w": 6, + "x": 6, + "y": 64 + }, + "id": 46, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "elasticsearch_thread_pool_active_count{cluster=\"$cluster\",name=~\"$node\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}} - {{ type }}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Thread Pool operations queued", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 19, + "w": 6, + "x": 12, + "y": 64 + }, + "height": "", + "id": 43, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "elasticsearch_thread_pool_active_count{cluster=\"$cluster\",name=~\"$node\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}} - {{ type }}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Thread Pool threads active", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 19, + "w": 6, + "x": 18, + "y": 64 + }, + "id": 44, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "irate(elasticsearch_thread_pool_completed_count{cluster=\"$cluster\",name=~\"$node\"}[$interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{name}} - {{ type }}", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Thread Pool operations completed", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 83 + }, + "id": 58, + "panels": [], + "repeat": null, + "title": "JVM Garbage Collection", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 10, + "w": 12, + "x": 0, + "y": 84 + }, + "height": "400", + "id": 7, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "rate(elasticsearch_jvm_gc_collection_seconds_count{cluster=\"$cluster\",name=~\"$node\"}[$interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}} - {{gc}}", + "metric": "", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "GC count", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "GCs", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 10, + "w": 12, + "x": 12, + "y": 84 + }, + "height": "400", + "id": 27, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "hideEmpty": false, + "hideZero": false, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(elasticsearch_jvm_gc_collection_seconds_count{cluster=\"$cluster\",name=~\"$node\"}[$interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "{{name}} - {{gc}}", + "metric": "", + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "GC time", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": "Time", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + } + ], + "refresh": "5m", + "schemaVersion": 18, + "style": "dark", + "tags": [ + "elasticsearch", + "App" + ], + "templating": { + "list": [ + { + "auto": true, + "auto_count": 30, + "auto_min": "10s", + "current": { + "text": "auto", + "value": "$__auto_interval_interval" + }, + "hide": 0, + "label": "Interval", + "name": "interval", + "options": [ + { + "selected": true, + "text": "auto", + "value": "$__auto_interval_interval" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + }, + { + "current": { + "text": "prometheus", + "value": "prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Prometheus datasource", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "", + "hide": 0, + "includeAll": false, + "label": "Instance", + "multi": false, + "name": "cluster", + "options": [], + "query": "label_values(elasticsearch_cluster_health_status,cluster)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": null, + "tags": [], + "tagsQuery": null, + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "${DS_PROMETHEUS}", + "definition": "", + "hide": 0, + "includeAll": true, + "label": "node", + "multi": true, + "name": "node", + "options": [], + "query": "label_values(elasticsearch_process_cpu_percent,name)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "tagValuesQuery": null, + "tags": [], + "tagsQuery": null, + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Elasticsearch", + "version": 1 + } +... diff --git a/values_overrides/grafana/home_dashboard.yaml b/values_overrides/grafana/home_dashboard.yaml new file mode 100644 index 0000000000..0e2b08964e --- /dev/null +++ b/values_overrides/grafana/home_dashboard.yaml @@ -0,0 +1,109 @@ +# This override file provides a reference for dashboards for +# customized OSH Welcome Page +--- +conf: + dashboards: + home: + home_dashboard: |- + { + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "gnetId": null, + "graphTooltip": 0, + "id": 66, + "links": [], + "panels": [ + { + "content": "
\n OSH Home Dashboard\n
", + "editable": true, + "gridPos": { + "h": 3, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "links": [], + "mode": "html", + "options": {}, + "style": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "folderId": 0, + "gridPos": { + "h": 10, + "w": 13, + "x": 6, + "y": 3 + }, + "headings": true, + "id": 3, + "limit": 30, + "links": [], + "options": {}, + "query": "", + "recent": true, + "search": false, + "starred": true, + "tags": [], + "title": "", + "type": "dashlist" + } + ], + "schemaVersion": 18, + "style": "dark", + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "hidden": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ], + "type": "timepicker" + }, + "timezone": "browser", + "title": "OSH Home", + "version": 1 + } +... diff --git a/values_overrides/grafana/kubernetes.yaml b/values_overrides/grafana/kubernetes.yaml new file mode 100644 index 0000000000..672e336b68 --- /dev/null +++ b/values_overrides/grafana/kubernetes.yaml @@ -0,0 +1,2116 @@ +# NOTE(srwilkers): This overrides file provides a reference for dashboards that +# reflect the overall state of a Kubernetes deployment +--- +conf: + dashboards: + kubernetes: + kubernetes_capacity_planning: |- + { + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "4.4.1" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "singlestat", + "name": "Singlestat", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "", + "overwrite": true, + "editable": false, + "gnetId": 22, + "graphTooltip": 0, + "id": 35, + "links": [], + "panels": [ + { + "alerting": {}, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_cpu{mode=\"idle\"}[2m])) * 100", + "hide": false, + "intervalFactor": 10, + "legendFormat": "", + "refId": "A", + "step": 50 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Idle cpu", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": "cpu usage", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "alerting": {}, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_load1)", + "intervalFactor": 4, + "legendFormat": "load 1m", + "refId": "A", + "step": 20, + "target": "" + }, + { + "expr": "sum(node_load5)", + "intervalFactor": 4, + "legendFormat": "load 5m", + "refId": "B", + "step": 20, + "target": "" + }, + { + "expr": "sum(node_load15)", + "intervalFactor": 4, + "legendFormat": "load 15m", + "refId": "C", + "step": 20, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "System load", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "alerting": {}, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 18, + "x": 0, + "y": 7 + }, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "node_memory_SwapFree{instance=\"172.17.0.1:9100\",job=\"prometheus\"}", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(node_memory_MemTotal) - sum(node_memory_MemFree) - sum(node_memory_Buffers) - sum(node_memory_Cached)", + "intervalFactor": 2, + "legendFormat": "memory usage", + "metric": "memo", + "refId": "A", + "step": 10, + "target": "" + }, + { + "expr": "sum(node_memory_Buffers)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "memory buffers", + "metric": "memo", + "refId": "B", + "step": 10, + "target": "" + }, + { + "expr": "sum(node_memory_Cached)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "memory cached", + "metric": "memo", + "refId": "C", + "step": 10, + "target": "" + }, + { + "expr": "sum(node_memory_MemFree)", + "interval": "", + "intervalFactor": 2, + "legendFormat": "memory free", + "metric": "memo", + "refId": "D", + "step": 10, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 7 + }, + "id": 5, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "((sum(node_memory_MemTotal) - sum(node_memory_MemFree) - sum(node_memory_Buffers) - sum(node_memory_Cached)) / sum(node_memory_MemTotal)) * 100", + "intervalFactor": 2, + "metric": "", + "refId": "A", + "step": 60, + "target": "" + } + ], + "thresholds": "80, 90", + "title": "Memory usage", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "alerting": {}, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 18, + "x": 0, + "y": 14 + }, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "read", + "yaxis": 1 + }, + { + "alias": "{instance=\"172.17.0.1:9100\"}", + "yaxis": 2 + }, + { + "alias": "io time", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_disk_bytes_read[5m]))", + "hide": false, + "intervalFactor": 4, + "legendFormat": "read", + "refId": "A", + "step": 20, + "target": "" + }, + { + "expr": "sum(rate(node_disk_bytes_written[5m]))", + "intervalFactor": 4, + "legendFormat": "written", + "refId": "B", + "step": 20 + }, + { + "expr": "sum(rate(node_disk_io_time_ms[5m]))", + "intervalFactor": 4, + "legendFormat": "io time", + "refId": "C", + "step": 20 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "format": "percentunit", + "gauge": { + "maxValue": 1, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 14 + }, + "id": 12, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(sum(node_filesystem_size{device!=\"rootfs\"}) - sum(node_filesystem_free{device!=\"rootfs\"})) / sum(node_filesystem_size{device!=\"rootfs\"})", + "intervalFactor": 2, + "refId": "A", + "step": 60, + "target": "" + } + ], + "thresholds": "0.75, 0.9", + "title": "Disk space usage", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "alerting": {}, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "transmitted ", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_receive_bytes{device!~\"lo\"}[5m]))", + "hide": false, + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 10, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network received", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "alerting": {}, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "transmitted ", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(node_network_transmit_bytes{device!~\"lo\"}[5m]))", + "hide": false, + "intervalFactor": 2, + "legendFormat": "", + "refId": "B", + "step": 10, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network transmitted", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fill": 1, + "gridPos": { + "h": 7, + "w": 18, + "x": 0, + "y": 28 + }, + "id": 11, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(kube_pod_info)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Current number of Pods", + "refId": "A", + "step": 10 + }, + { + "expr": "sum(kube_node_status_capacity_pods)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Maximum capacity of pods", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cluster Pod Utilization", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 28 + }, + "id": 7, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "100 - (sum(kube_node_status_capacity_pods) - sum(kube_pod_info)) / sum(kube_node_status_capacity_pods) * 100", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 60, + "target": "" + } + ], + "thresholds": "80,90", + "title": "Pod Utilization", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "refresh": false, + "schemaVersion": 18, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "prometheus", + "value": "prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Prometheus datasource", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Kubernetes Capacity Planning", + "version": 1 + } + kubernetes_cluster_status: |- + { + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "4.4.1" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "singlestat", + "name": "Singlestat", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": false, + "overwrite": true, + "gnetId": null, + "graphTooltip": 0, + "id": 5, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 11, + "panels": [], + "repeat": null, + "title": "Cluster Health", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "${DS_PROMETHEUS}", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 0, + "y": 1 + }, + "id": 5, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(up{job=~\"apiserver|kube-scheduler|kube-controller-manager\"} == 0)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 600 + } + ], + "thresholds": "1,3", + "title": "Control Plane UP", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "UP", + "value": "null" + } + ], + "valueName": "total" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "${DS_PROMETHEUS}", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 1 + }, + "id": 6, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(ALERTS{alertstate=\"firing\",alertname!=\"DeadMansSwitch\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 600 + } + ], + "thresholds": "3,5", + "title": "Alerts Firing", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 12, + "panels": [], + "repeat": null, + "title": "Control Plane Status", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_PROMETHEUS}", + "decimals": null, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 6 + }, + "id": 1, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(sum(up{job=\"apiserver\"} == 1) / count(up{job=\"apiserver\"})) * 100", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 600 + } + ], + "thresholds": "50,80", + "title": "API Servers UP", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_PROMETHEUS}", + "decimals": null, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 6 + }, + "id": 2, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(sum(up{job=\"kube-controller-manager-discovery\"} == 1) / count(up{job=\"kube-controller-manager-discovery\"})) * 100", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 600 + } + ], + "thresholds": "50,80", + "title": "Controller Managers UP", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_PROMETHEUS}", + "decimals": null, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 6 + }, + "id": 3, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(sum(up{job=\"kube-scheduler-discovery\"} == 1) / count(up{job=\"kube-scheduler-discovery\"})) * 100", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 600 + } + ], + "thresholds": "50,80", + "title": "Schedulers UP", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "${DS_PROMETHEUS}", + "decimals": null, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 6 + }, + "hideTimeOverride": false, + "id": 4, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "count(increase(kube_pod_container_status_restarts{namespace=~\"kube-system|tectonic-system\"}[1h]) > 5)", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 600 + } + ], + "thresholds": "1,3", + "title": "Crashlooping Control Plane Pods", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 11 + }, + "id": 13, + "panels": [], + "repeat": null, + "title": "Capacity Planing", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "${DS_PROMETHEUS}", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 12 + }, + "id": 8, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(100 - (avg by (instance) (rate(node_cpu{job=\"node-exporter\",mode=\"idle\"}[5m])) * 100)) / count(node_cpu{job=\"node-exporter\",mode=\"idle\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 600 + } + ], + "thresholds": "80,90", + "title": "CPU Utilization", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "${DS_PROMETHEUS}", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 12 + }, + "id": 7, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "((sum(node_memory_MemTotal) - sum(node_memory_MemFree) - sum(node_memory_Buffers) - sum(node_memory_Cached)) / sum(node_memory_MemTotal)) * 100", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 600 + } + ], + "thresholds": "80,90", + "title": "Memory Utilization", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "${DS_PROMETHEUS}", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 12 + }, + "id": 9, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(sum(node_filesystem_size{device!=\"rootfs\"}) - sum(node_filesystem_free{device!=\"rootfs\"})) / sum(node_filesystem_size{device!=\"rootfs\"})", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 600 + } + ], + "thresholds": "80,90", + "title": "Filesystem Utilization", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "${DS_PROMETHEUS}", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 12 + }, + "id": 10, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "100 - (sum(kube_node_status_capacity_pods) - sum(kube_pod_info)) / sum(kube_node_status_capacity_pods) * 100", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A", + "step": 600 + } + ], + "thresholds": "80,90", + "title": "Pod Utilization", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + } + ], + "schemaVersion": 18, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "prometheus", + "value": "prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Prometheus datasource", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Kubernetes Cluster Status", + "version": 1 + } +... diff --git a/values_overrides/grafana/nginx.yaml b/values_overrides/grafana/nginx.yaml new file mode 100644 index 0000000000..5cc2504d40 --- /dev/null +++ b/values_overrides/grafana/nginx.yaml @@ -0,0 +1,1467 @@ +# NOTE(srwilkers): This overrides file provides a reference for a dashboard for +# nginx +--- +conf: + dashboards: + kubernetes: + nginx_stats: |- + { + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "5.0.0" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "5.0.0" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "panel", + "id": "singlestat", + "name": "Singlestat", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": "${DS_PROMETHEUS}", + "enable": true, + "expr": "sum(changes(nginx_ingress_controller_config_last_reload_successful_timestamp_seconds{instance!=\"unknown\",controller_class=~\"$controller_class\",namespace=~\"$namespace\"}[30s])) by (controller_class)", + "hide": false, + "iconColor": "rgba(255, 96, 96, 1)", + "limit": 100, + "name": "Config Reloads", + "showIn": 0, + "step": "30s", + "tagKeys": "controller_class", + "tags": [], + "titleFormat": "Config Reloaded", + "type": "tags" + } + ] + }, + "editable": true, + "overwrite": true, + "gnetId": null, + "graphTooltip": 0, + "links": [], + "panels": [ + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_PROMETHEUS}", + "format": "ops", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 20, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "round(sum(irate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\"}[2m])), 0.001)", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Controller Request Volume", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_PROMETHEUS}", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 82, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(avg_over_time(nginx_ingress_controller_nginx_process_connections{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m]))", + "format": "time_series", + "instant": false, + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Controller Connections", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_PROMETHEUS}", + "format": "percentunit", + "gauge": { + "maxValue": 100, + "minValue": 80, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": false + }, + "gridPos": { + "h": 3, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 21, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\",status!~\"[4-5].*\"}[2m])) / sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\"}[2m]))", + "format": "time_series", + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "95, 99, 99.5", + "title": "Controller Success Rate (non-4|5xx responses)", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_PROMETHEUS}", + "decimals": 0, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 18, + "y": 0 + }, + "id": 81, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "avg(nginx_ingress_controller_success{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"})", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Config Reloads", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "datasource": "${DS_PROMETHEUS}", + "decimals": 0, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 21, + "y": 0 + }, + "id": 83, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": true, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "count(nginx_ingress_controller_config_last_reload_successful{controller_pod=~\"$controller\",controller_namespace=~\"$namespace\"} == 0)", + "format": "time_series", + "instant": true, + "intervalFactor": 1, + "refId": "A", + "step": 4 + } + ], + "thresholds": "", + "title": "Last Config Failed", + "transparent": false, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "None", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 3 + }, + "height": "200px", + "id": 86, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": false, + "hideZero": true, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 300, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "repeatDirection": "h", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "round(sum(irate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress), 0.001)", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "metric": "network", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Ingress Request Volume", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "max - istio-proxy": "#890f02", + "max - master": "#bf1b00", + "max - prometheus": "#bf1b00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "editable": false, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 3 + }, + "id": 87, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "hideEmpty": true, + "hideZero": false, + "max": false, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 300, + "sort": "avg", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\",ingress=~\"$ingress\",status!~\"[4-5].*\"}[2m])) by (ingress) / sum(rate(nginx_ingress_controller_requests{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress)", + "format": "time_series", + "instant": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "metric": "container_memory_usage:sort_desc", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Ingress Success Rate (non-4|5xx responses)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 1, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 10 + }, + "height": "200px", + "id": 32, + "isNew": true, + "legend": { + "alignAsTable": false, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (irate (nginx_ingress_controller_request_size_sum{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m]))", + "format": "time_series", + "instant": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "Received", + "metric": "network", + "refId": "A", + "step": 10 + }, + { + "expr": "- sum (irate (nginx_ingress_controller_response_size_sum{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m]))", + "format": "time_series", + "hide": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "Sent", + "metric": "network", + "refId": "B", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Network I/O pressure", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "Bps", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "max - istio-proxy": "#890f02", + "max - master": "#bf1b00", + "max - prometheus": "#bf1b00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "editable": false, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 10 + }, + "id": 77, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sideWidth": 200, + "sort": "current", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "avg(nginx_ingress_controller_nginx_process_resident_memory_bytes{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}) ", + "format": "time_series", + "instant": false, + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "nginx", + "metric": "container_memory_usage:sort_desc", + "refId": "A", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Average Memory Usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "max - istio-proxy": "#890f02", + "max - master": "#bf1b00" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 3, + "editable": false, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 10 + }, + "height": "", + "id": 79, + "isNew": true, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "sort": null, + "sortDesc": null, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum (rate (nginx_ingress_controller_nginx_process_cpu_seconds_total{controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\"}[2m])) ", + "format": "time_series", + "interval": "10s", + "intervalFactor": 1, + "legendFormat": "nginx", + "metric": "container_cpu", + "refId": "A", + "step": 10 + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "gt" + } + ], + "timeFrom": null, + "timeShift": null, + "title": "Average CPU Usage", + "tooltip": { + "msResolution": true, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": "cores", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "columns": [], + "datasource": "${DS_PROMETHEUS}", + "fontSize": "100%", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 16 + }, + "hideTimeOverride": false, + "id": 75, + "links": [], + "pageSize": 7, + "repeat": null, + "repeatDirection": "h", + "scroll": true, + "showHeader": true, + "sort": { + "col": 1, + "desc": true + }, + "styles": [ + { + "alias": "Ingress", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "ingress", + "preserveFormat": false, + "sanitize": false, + "thresholds": [], + "type": "string", + "unit": "short" + }, + { + "alias": "Requests", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #A", + "thresholds": [ + "" + ], + "type": "number", + "unit": "ops" + }, + { + "alias": "Errors", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #B", + "thresholds": [], + "type": "number", + "unit": "ops" + }, + { + "alias": "P50 Latency", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "link": false, + "pattern": "Value #C", + "thresholds": [], + "type": "number", + "unit": "dtdurations" + }, + { + "alias": "P90 Latency", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "pattern": "Value #D", + "thresholds": [], + "type": "number", + "unit": "dtdurations" + }, + { + "alias": "P99 Latency", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "pattern": "Value #E", + "thresholds": [], + "type": "number", + "unit": "dtdurations" + }, + { + "alias": "IN", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Value #F", + "thresholds": [ + "" + ], + "type": "number", + "unit": "Bps" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "pattern": "Time", + "thresholds": [], + "type": "hidden", + "unit": "short" + }, + { + "alias": "OUT", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 2, + "mappingType": 1, + "pattern": "Value #G", + "thresholds": [], + "type": "number", + "unit": "Bps" + } + ], + "targets": [ + { + "expr": "histogram_quantile(0.50, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (le, ingress))", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "refId": "C" + }, + { + "expr": "histogram_quantile(0.90, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (le, ingress))", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "refId": "D" + }, + { + "expr": "histogram_quantile(0.99, sum(rate(nginx_ingress_controller_request_duration_seconds_bucket{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (le, ingress))", + "format": "table", + "hide": false, + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ destination_service }}", + "refId": "E" + }, + { + "expr": "sum(irate(nginx_ingress_controller_request_size_sum{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress)", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "refId": "F" + }, + { + "expr": "sum(irate(nginx_ingress_controller_response_size_sum{ingress!=\"\",controller_pod=~\"$controller\",controller_class=~\"$controller_class\",controller_namespace=~\"$namespace\",ingress=~\"$ingress\"}[2m])) by (ingress)", + "format": "table", + "instant": true, + "intervalFactor": 1, + "legendFormat": "{{ ingress }}", + "refId": "G" + } + ], + "timeFrom": null, + "title": "Ingress Percentile Response Times and Transfer Rates", + "transform": "table", + "transparent": false, + "type": "table" + }, + { + "columns": [ + { + "text": "Current", + "value": "current" + } + ], + "datasource": "${DS_PROMETHEUS}", + "fontSize": "100%", + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 24 + }, + "height": "1024", + "id": 85, + "links": [], + "pageSize": 7, + "scroll": true, + "showHeader": true, + "sort": { + "col": 1, + "desc": false + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "Time", + "type": "date" + }, + { + "alias": "TTL", + "colorMode": "cell", + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "decimals": 0, + "pattern": "Current", + "thresholds": [ + "0", + "691200" + ], + "type": "number", + "unit": "s" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "expr": "avg(nginx_ingress_controller_ssl_expire_time_seconds{kubernetes_pod_name=~\"$controller\",namespace=~\"$namespace\",ingress=~\"$ingress\"}) by (host) - time()", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "{{ host }}", + "metric": "gke_letsencrypt_cert_expiration", + "refId": "A", + "step": 1 + } + ], + "title": "Ingress Certificate Expiry", + "transform": "timeseries_aggregations", + "type": "table" + } + ], + "refresh": "5s", + "schemaVersion": 16, + "style": "dark", + "tags": [ + "nginx" + ], + "templating": { + "list": [ + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "${DS_PROMETHEUS}", + "hide": 0, + "includeAll": true, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [], + "query": "label_values(nginx_ingress_controller_config_hash, controller_namespace)", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "${DS_PROMETHEUS}", + "hide": 0, + "includeAll": true, + "label": "Controller Class", + "multi": false, + "name": "controller_class", + "options": [], + "query": "label_values(nginx_ingress_controller_config_hash{namespace=~\"$namespace\"}, controller_class) ", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "text": "All", + "value": "$__all" + }, + "datasource": "${DS_PROMETHEUS}", + "hide": 0, + "includeAll": true, + "label": "Controller", + "multi": false, + "name": "controller", + "options": [], + "query": "label_values(nginx_ingress_controller_config_hash{namespace=~\"$namespace\",controller_class=~\"$controller_class\"}, controller_pod) ", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": ".*", + "current": { + "tags": [], + "text": "All", + "value": "$__all" + }, + "datasource": "${DS_PROMETHEUS}", + "hide": 0, + "includeAll": true, + "label": "Ingress", + "multi": false, + "name": "ingress", + "options": [], + "query": "label_values(nginx_ingress_controller_requests{namespace=~\"$namespace\",controller_class=~\"$controller_class\",controller=~\"$controller\"}, ingress) ", + "refresh": 1, + "regex": "", + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "2m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "NGINX Ingress controller", + "uid": "nginx", + "version": 1 + } +... diff --git a/values_overrides/grafana/nodes.yaml b/values_overrides/grafana/nodes.yaml new file mode 100644 index 0000000000..d7542f3af8 --- /dev/null +++ b/values_overrides/grafana/nodes.yaml @@ -0,0 +1,981 @@ +# NOTE(srwilkers): This overrides file provides a reference for a dashboard for +# the status of all nodes in a deployment +--- +conf: + dashboards: + lma: + nodes: |- + { + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "4.4.1" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "singlestat", + "name": "Singlestat", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "Dashboard to get an overview of one server", + "overwrite": true, + "editable": true, + "gnetId": 22, + "graphTooltip": 0, + "id": 8, + "links": [], + "panels": [ + { + "alerting": {}, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "100 - (avg by (cpu) (irate(node_cpu{mode=\"idle\", instance=\"$server\"}[5m])) * 100)", + "hide": false, + "intervalFactor": 10, + "legendFormat": "{{cpu}}", + "refId": "A", + "step": 50 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Idle cpu", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percent", + "label": "cpu usage", + "logBase": 1, + "max": 100, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "alerting": {}, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 9, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "node_load1{instance=\"$server\"}", + "intervalFactor": 4, + "legendFormat": "load 1m", + "refId": "A", + "step": 20, + "target": "" + }, + { + "expr": "node_load5{instance=\"$server\"}", + "intervalFactor": 4, + "legendFormat": "load 5m", + "refId": "B", + "step": 20, + "target": "" + }, + { + "expr": "node_load15{instance=\"$server\"}", + "intervalFactor": 4, + "legendFormat": "load 15m", + "refId": "C", + "step": 20, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "System load", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "percentunit", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "alerting": {}, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 18, + "x": 0, + "y": 7 + }, + "id": 4, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "hideZero": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "node_memory_SwapFree{instance=\"$server\",job=\"prometheus\"}", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "node_memory_MemTotal{instance=\"$server\"} - node_memory_MemFree{instance=\"$server\"} - node_memory_Buffers{instance=\"$server\"} - node_memory_Cached{instance=\"$server\"}", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "memory used", + "metric": "", + "refId": "C", + "step": 10 + }, + { + "expr": "node_memory_Buffers{instance=\"$server\"}", + "interval": "", + "intervalFactor": 2, + "legendFormat": "memory buffers", + "metric": "", + "refId": "E", + "step": 10 + }, + { + "expr": "node_memory_Cached{instance=\"$server\"}", + "intervalFactor": 2, + "legendFormat": "memory cached", + "metric": "", + "refId": "F", + "step": 10 + }, + { + "expr": "node_memory_MemFree{instance=\"$server\"}", + "intervalFactor": 2, + "legendFormat": "memory free", + "metric": "", + "refId": "D", + "step": 10 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Memory usage", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 7 + }, + "id": 5, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "((node_memory_MemTotal{instance=\"$server\"} - node_memory_MemFree{instance=\"$server\"} - node_memory_Buffers{instance=\"$server\"} - node_memory_Cached{instance=\"$server\"}) / node_memory_MemTotal{instance=\"$server\"}) * 100", + "intervalFactor": 2, + "refId": "A", + "step": 60, + "target": "" + } + ], + "thresholds": "80, 90", + "title": "Memory usage", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "alerting": {}, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 18, + "x": 0, + "y": 14 + }, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "read", + "yaxis": 1 + }, + { + "alias": "{instance=\"$server\"}", + "yaxis": 2 + }, + { + "alias": "io time", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (instance) (rate(node_disk_bytes_read{instance=\"$server\"}[2m]))", + "hide": false, + "intervalFactor": 4, + "legendFormat": "read", + "refId": "A", + "step": 20, + "target": "" + }, + { + "expr": "sum by (instance) (rate(node_disk_bytes_written{instance=\"$server\"}[2m]))", + "intervalFactor": 4, + "legendFormat": "written", + "refId": "B", + "step": 20 + }, + { + "expr": "sum by (instance) (rate(node_disk_io_time_ms{instance=\"$server\"}[2m]))", + "intervalFactor": 4, + "legendFormat": "io time", + "refId": "C", + "step": 20 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk I/O", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "format": "percentunit", + "gauge": { + "maxValue": 1, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 14 + }, + "id": 7, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(sum(node_filesystem_size{device!=\"rootfs\",instance=\"$server\"}) - sum(node_filesystem_free{device!=\"rootfs\",instance=\"$server\"})) / sum(node_filesystem_size{device!=\"rootfs\",instance=\"$server\"})", + "intervalFactor": 2, + "refId": "A", + "step": 60, + "target": "" + } + ], + "thresholds": "0.75, 0.9", + "title": "Disk space usage", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "alerting": {}, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "transmitted ", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_receive_bytes{instance=\"$server\",device!~\"lo\"}[5m])", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{device}}", + "refId": "A", + "step": 10, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network received", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "alerting": {}, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 10, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "transmitted ", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(node_network_transmit_bytes{instance=\"$server\",device!~\"lo\"}[5m])", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{device}}", + "refId": "B", + "step": 10, + "target": "" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Network transmitted", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": false, + "schemaVersion": 18, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "prometheus", + "value": "prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Prometheus datasource", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "", + "hide": 0, + "includeAll": false, + "label": "Server", + "multi": false, + "name": "host", + "options": [], + "query": "label_values(node_uname_info, nodename)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "", + "hide": 2, + "includeAll": false, + "label": "Instance", + "multi": false, + "name": "server", + "options": [], + "query": "label_values(node_uname_info{nodename=\"$host\"}, instance)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Nodes", + "version": 1 + } +... diff --git a/values_overrides/grafana/openstack.yaml b/values_overrides/grafana/openstack.yaml new file mode 100644 index 0000000000..d80847724f --- /dev/null +++ b/values_overrides/grafana/openstack.yaml @@ -0,0 +1,4165 @@ +# NOTE(srwilkers): This overrides file provides a reference for dashboards for +# the openstack control plane as a whole, the individual openstack services, and +# rabbitmq +--- +conf: + dashboards: + openstack: + rabbitmq: |- + { + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "4.2.0" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "singlestat", + "name": "Singlestat", + "version": "" + } + ], + "annotations": { + "list": [] + }, + "editable": true, + "overwrite": true, + "gnetId": 2121, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [], + "refresh": "5s", + "rows": [ + { + "collapse": false, + "height": 266, + "panels": [ + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "${DS_PROMETHEUS}", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "id": 13, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "targets": [ + { + "expr": "rabbitmq_up", + "intervalFactor": 2, + "metric": "rabbitmq_up", + "refId": "A", + "step": 2 + } + ], + "thresholds": "Up,Down", + "timeFrom": "30s", + "title": "RabbitMQ Server", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + }, + { + "op": "=", + "text": "Down", + "value": "0" + }, + { + "op": "=", + "text": "Up", + "value": "1" + } + ], + "valueName": "current" + }, + { + "alert": { + "conditions": [ + { + "evaluator": { + "params": [ + 1 + ], + "type": "lt" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "A", + "10s", + "now" + ] + }, + "reducer": { + "params": [], + "type": "last" + }, + "type": "query" + }, + { + "evaluator": { + "params": [], + "type": "no_value" + }, + "operator": { + "type": "and" + }, + "query": { + "params": [ + "A", + "10s", + "now" + ] + }, + "reducer": { + "params": [], + "type": "last" + }, + "type": "query" + } + ], + "executionErrorState": "alerting", + "frequency": "60s", + "handler": 1, + "message": "Some of the RabbitMQ node is down", + "name": "Node Stats alert", + "noDataState": "no_data", + "notifications": [] + }, + "aliasColors": {}, + "bars": true, + "datasource": "${DS_PROMETHEUS}", + "decimals": 0, + "fill": 1, + "id": 12, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "show": true, + "total": false, + "values": true + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 9, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rabbitmq_running", + "intervalFactor": 2, + "legendFormat": "{{node}}", + "metric": "rabbitmq_running", + "refId": "A", + "step": 2 + } + ], + "thresholds": [ + { + "colorMode": "critical", + "fill": true, + "line": true, + "op": "lt", + "value": 1 + } + ], + "timeFrom": "30s", + "timeShift": null, + "title": "Node up Stats", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 0, + "fill": 1, + "id": 6, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rabbitmq_exchangesTotal", + "intervalFactor": 2, + "legendFormat": "{{instance}}:exchanges", + "metric": "rabbitmq_exchangesTotal", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Exchanges", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 0, + "fill": 1, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rabbitmq_channelsTotal", + "intervalFactor": 2, + "legendFormat": "{{instance}}:channels", + "metric": "rabbitmq_channelsTotal", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Channels", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 0, + "fill": 1, + "id": 3, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rabbitmq_consumersTotal", + "intervalFactor": 2, + "legendFormat": "{{instance}}:consumers", + "metric": "rabbitmq_consumersTotal", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Consumers", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 0, + "fill": 1, + "id": 5, + "legend": { + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rabbitmq_connectionsTotal", + "intervalFactor": 2, + "legendFormat": "{{instance}}:connections", + "metric": "rabbitmq_connectionsTotal", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Connections", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_PROMETHEUS}", + "fill": 1, + "id": 7, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 4, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rabbitmq_queuesTotal", + "intervalFactor": 2, + "legendFormat": "{{instance}}:queues", + "metric": "rabbitmq_queuesTotal", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Queues", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 0, + "fill": 1, + "id": 8, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum by (vhost)(rabbitmq_queue_messages_ready)", + "intervalFactor": 2, + "legendFormat": "{{vhost}}:ready", + "metric": "rabbitmq_queue_messages_ready", + "refId": "A", + "step": 2 + }, + { + "expr": "sum by (vhost)(rabbitmq_queue_messages_published_total)", + "intervalFactor": 2, + "legendFormat": "{{vhost}}:published", + "metric": "rabbitmq_queue_messages_published_total", + "refId": "B", + "step": 2 + }, + { + "expr": "sum by (vhost)(rabbitmq_queue_messages_delivered_total)", + "intervalFactor": 2, + "legendFormat": "{{vhost}}:delivered", + "metric": "rabbitmq_queue_messages_delivered_total", + "refId": "C", + "step": 2 + }, + { + "expr": "sum by (vhost)(rabbitmq_queue_messages_unacknowledged)", + "intervalFactor": 2, + "legendFormat": "{{vhost}}:unack", + "metric": "ack", + "refId": "D", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Messages/host", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 0, + "fill": 1, + "id": 2, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rabbitmq_queue_messages", + "intervalFactor": 2, + "legendFormat": "{{queue}}:{{durable}}", + "metric": "rabbitmq_queue_messages", + "refId": "A", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Messages / Queue", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_PROMETHEUS}", + "fill": 1, + "id": 9, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rabbitmq_node_mem_used", + "intervalFactor": 2, + "legendFormat": "{{node}}:used", + "metric": "rabbitmq_node_mem_used", + "refId": "A", + "step": 2 + }, + { + "expr": "rabbitmq_node_mem_limit", + "intervalFactor": 2, + "legendFormat": "{{node}}:limit", + "metric": "node_mem", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "decbytes", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_PROMETHEUS}", + "fill": 1, + "id": 10, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rabbitmq_fd_used", + "intervalFactor": 2, + "legendFormat": "{{node}}:used", + "metric": "", + "refId": "A", + "step": 2 + }, + { + "expr": "rabbitmq_fd_total", + "intervalFactor": 2, + "legendFormat": "{{node}}:total", + "metric": "node_mem", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "FIle descriptors", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "datasource": "${DS_PROMETHEUS}", + "fill": 1, + "id": 11, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rabbitmq_sockets_used", + "intervalFactor": 2, + "legendFormat": "{{node}}:used", + "metric": "", + "refId": "A", + "step": 2 + }, + { + "expr": "rabbitmq_sockets_total", + "intervalFactor": 2, + "legendFormat": "{{node}}:total", + "metric": "", + "refId": "B", + "step": 2 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Sockets", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "transparent": false, + "type": "graph", + "xaxis": { + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "tags": [], + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "RabbitMQ Metrics", + "version": 17, + "description": "Basic rabbitmq host stats: Node Stats, Exchanges, Channels, Consumers, Connections, Queues, Messages, Messages per Queue, Memory, File Descriptors, Sockets." + } + openstack_control_plane: |- + { + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "4.5.2" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "singlestat", + "name": "Singlestat", + "version": "" + }, + { + "type": "panel", + "id": "text", + "name": "Text", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": false, + "overwrite": true, + "gnetId": null, + "graphTooltip": 1, + "id": 11, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 28, + "panels": [], + "repeat": null, + "title": "OpenStack Services", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "rgba(200, 54, 35, 0.88)", + "rgba(118, 245, 40, 0.73)", + "rgba(225, 177, 40, 0.59)" + ], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 2, + "x": 0, + "y": 1 + }, + "id": 24, + "interval": "> 60s", + "links": [ + { + "dashboard": "Openstack Service", + "name": "Drilldown dashboard", + "params": "var-Service=keystone", + "title": "Openstack Service", + "type": "dashboard" + } + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "column": "value", + "condition": "", + "expr": "openstack_check_keystone_api{job=\"openstack-metrics\", region=\"$region\"}", + "fill": "", + "format": "time_series", + "function": "last", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "groupByTags": [], + "groupby_field": "", + "interval": "", + "intervalFactor": 2, + "policy": "default", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "step": 120 + } + ], + "thresholds": "1,2", + "title": "Keystone", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "no data", + "value": "null" + }, + { + "op": "=", + "text": "CRIT", + "value": "0" + }, + { + "op": "=", + "text": "OK", + "value": "1" + }, + { + "op": "=", + "text": "UNKW", + "value": "2" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "rgba(200, 54, 35, 0.88)", + "rgba(118, 245, 40, 0.73)", + "rgba(225, 177, 40, 0.59)" + ], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 2, + "x": 2, + "y": 1 + }, + "id": 23, + "interval": "> 60s", + "links": [ + { + "dashboard": "Openstack Service", + "name": "Drilldown dashboard", + "params": "var-Service=glance", + "title": "Openstack Service", + "type": "dashboard" + } + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "column": "value", + "condition": "", + "expr": "openstack_check_glance_api{job=\"openstack-metrics\", region=\"$region\"}", + "fill": "", + "format": "time_series", + "function": "last", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "groupByTags": [], + "groupby_field": "", + "interval": "", + "intervalFactor": 2, + "policy": "default", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "step": 120 + } + ], + "thresholds": "1,2", + "title": "Glance", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "no data", + "value": "null" + }, + { + "op": "=", + "text": "CRIT", + "value": "0" + }, + { + "op": "=", + "text": "OK", + "value": "1" + }, + { + "op": "=", + "text": "UNKW", + "value": "2" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "rgba(202, 58, 40, 0.86)", + "rgba(118, 245, 40, 0.73)", + "rgba(225, 177, 40, 0.59)" + ], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 2, + "x": 4, + "y": 1 + }, + "id": 22, + "interval": "> 60s", + "links": [ + { + "dashboard": "Openstack Service", + "name": "Drilldown dashboard", + "params": "var-Service=heat", + "title": "Openstack Service", + "type": "dashboard" + } + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "column": "value", + "condition": "", + "expr": "openstack_check_heat_api{job=\"openstack-metrics\", region=\"$region\"}", + "fill": "", + "format": "time_series", + "function": "last", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "groupByTags": [], + "groupby_field": "", + "interval": "", + "intervalFactor": 2, + "policy": "default", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "step": 120 + } + ], + "thresholds": "1,2", + "title": "Heat", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "no data", + "value": "null" + }, + { + "op": "=", + "text": "CRIT", + "value": "0" + }, + { + "op": "=", + "text": "OK", + "value": "1" + }, + { + "op": "=", + "text": "UNKW", + "value": "2" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "rgba(200, 54, 35, 0.88)", + "rgba(118, 245, 40, 0.73)", + "rgba(225, 177, 40, 0.59)" + ], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 2, + "x": 6, + "y": 1 + }, + "id": 21, + "interval": "> 60s", + "links": [ + { + "dashboard": "Openstack Service", + "name": "Drilldown dashboard", + "params": "var-Service=neutron", + "title": "Openstack Service", + "type": "dashboard" + } + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "column": "value", + "condition": "", + "expr": "openstack_check_neutron_api{job=\"openstack-metrics\", region=\"$region\"}", + "fill": "", + "format": "time_series", + "function": "last", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "groupByTags": [], + "groupby_field": "", + "interval": "", + "intervalFactor": 2, + "policy": "default", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "step": 120 + } + ], + "thresholds": "1,2", + "title": "Neutron", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "no data", + "value": "null" + }, + { + "op": "=", + "text": "CRIT", + "value": "0" + }, + { + "op": "=", + "text": "OK", + "value": "1" + }, + { + "op": "=", + "text": "UNKW", + "value": "2" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "rgba(208, 53, 34, 0.82)", + "rgba(118, 245, 40, 0.73)", + "rgba(225, 177, 40, 0.59)" + ], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 2, + "x": 8, + "y": 1 + }, + "id": 20, + "interval": "> 60s", + "links": [ + { + "dashboard": "Openstack Service", + "name": "Drilldown dashboard", + "params": "var-Service=nova", + "title": "Openstack Service", + "type": "dashboard" + } + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "column": "value", + "condition": "", + "expr": "openstack_check_nova_api{job=\"openstack-metrics\", region=\"$region\"}", + "fill": "", + "format": "time_series", + "function": "last", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "groupByTags": [], + "groupby_field": "", + "interval": "", + "intervalFactor": 2, + "policy": "default", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "step": 120 + } + ], + "thresholds": "1,2", + "title": "Nova", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "no data", + "value": "null" + }, + { + "op": "=", + "text": "CRIT", + "value": "0" + }, + { + "op": "=", + "text": "OK", + "value": "1" + }, + { + "op": "=", + "text": "UNKW", + "value": "2" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "rgba(200, 54, 35, 0.88)", + "rgba(118, 245, 40, 0.73)", + "rgba(225, 177, 40, 0.59)" + ], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 2, + "x": 10, + "y": 1 + }, + "id": 19, + "interval": "> 60s", + "links": [ + { + "dashboard": "Openstack Service", + "name": "Drilldown dashboard", + "params": "var-Service=swift", + "title": "Openstack Service", + "type": "dashboard" + } + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "column": "value", + "condition": "", + "expr": "openstack_check_swift_api{job=\"openstack-metrics\", region=\"$region\"}", + "fill": "", + "format": "time_series", + "function": "last", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "groupByTags": [], + "groupby_field": "", + "interval": "", + "intervalFactor": 2, + "policy": "default", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "step": 120 + } + ], + "thresholds": "1,2", + "title": "Ceph", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "no data", + "value": "null" + }, + { + "op": "=", + "text": "CRIT", + "value": "0" + }, + { + "op": "=", + "text": "OK", + "value": "1" + }, + { + "op": "=", + "text": "UNKW", + "value": "2" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "rgba(200, 54, 35, 0.88)", + "rgba(118, 245, 40, 0.73)", + "rgba(225, 177, 40, 0.59)" + ], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 2, + "x": 12, + "y": 1 + }, + "id": 18, + "interval": "> 60s", + "links": [ + { + "dashboard": "Openstack Service", + "name": "Drilldown dashboard", + "params": "var-Service=cinder", + "title": "Openstack Service", + "type": "dashboard" + } + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "column": "value", + "condition": "", + "expr": "openstack_check_cinder_api{job=\"openstack-metrics\", region=\"$region\"}", + "fill": "", + "format": "time_series", + "function": "last", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "groupByTags": [], + "groupby_field": "", + "interval": "", + "intervalFactor": 2, + "policy": "default", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "step": 120 + } + ], + "thresholds": "1,2", + "title": "Cinder", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "no data", + "value": "null" + }, + { + "op": "=", + "text": "CRIT", + "value": "0" + }, + { + "op": "=", + "text": "OK", + "value": "1" + }, + { + "op": "=", + "text": "UNKW", + "value": "2" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "rgba(200, 54, 35, 0.88)", + "rgba(118, 245, 40, 0.73)", + "rgba(225, 177, 40, 0.59)" + ], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 2, + "x": 14, + "y": 1 + }, + "id": 17, + "interval": "> 60s", + "links": [ + { + "dashboard": "Openstack Service", + "name": "Drilldown dashboard", + "params": "var-Service=placement", + "title": "Openstack Service", + "type": "dashboard" + } + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "column": "value", + "condition": "", + "expr": "openstack_check_placement_api{job=\"openstack-metrics\", region=\"$region\"}", + "fill": "", + "format": "time_series", + "function": "last", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "groupByTags": [], + "groupby_field": "", + "interval": "", + "intervalFactor": 2, + "policy": "default", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "step": 120 + } + ], + "thresholds": "1,2", + "title": "Placement", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "no data", + "value": "null" + }, + { + "op": "=", + "text": "CRIT", + "value": "0" + }, + { + "op": "=", + "text": "OK", + "value": "1" + }, + { + "op": "=", + "text": "UNKW", + "value": "2" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "rgba(208, 53, 34, 0.82)", + "rgba(118, 245, 40, 0.73)", + "rgba(225, 177, 40, 0.59)" + ], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 2, + "x": 16, + "y": 1 + }, + "id": 16, + "interval": "> 60s", + "links": [ + { + "dashboard": "RabbitMQ Metrics", + "name": "Drilldown dashboard", + "title": "RabbitMQ Metrics", + "type": "dashboard" + } + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "column": "value", + "condition": "", + "expr": "min(rabbitmq_up)", + "fill": "", + "format": "time_series", + "function": "last", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "groupByTags": [], + "groupby_field": "", + "interval": "", + "intervalFactor": 2, + "policy": "default", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "step": 120 + } + ], + "thresholds": "1,2", + "title": "RabbitMQ", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "no data", + "value": "null" + }, + { + "op": "=", + "text": "CRIT", + "value": "0" + }, + { + "op": "=", + "text": "OK", + "value": "1" + }, + { + "op": "=", + "text": "UNKW", + "value": "2" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "rgba(208, 53, 34, 0.82)", + "rgba(118, 245, 40, 0.73)", + "rgba(225, 177, 40, 0.59)" + ], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 2, + "x": 18, + "y": 1 + }, + "id": 15, + "interval": "> 60s", + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "column": "value", + "condition": "", + "expr": "min(mysql_global_status_wsrep_ready)", + "fill": "", + "format": "time_series", + "function": "last", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "groupByTags": [], + "groupby_field": "", + "interval": "", + "intervalFactor": 2, + "policy": "default", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "step": 120 + } + ], + "thresholds": "1,2", + "title": "MariaDB", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "no data", + "value": "null" + }, + { + "op": "=", + "text": "CRIT", + "value": "0" + }, + { + "op": "=", + "text": "OK", + "value": "1" + }, + { + "op": "=", + "text": "UNKW", + "value": "2" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "rgba(225, 177, 40, 0.59)", + "rgba(208, 53, 34, 0.82)", + "rgba(118, 245, 40, 0.73)" + ], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 2, + "x": 20, + "y": 1 + }, + "id": 14, + "interval": "> 60s", + "links": [ + { + "dashboard": "Nginx Stats", + "name": "Drilldown dashboard", + "title": "Nginx Stats", + "type": "dashboard" + } + ], + "mappingType": 2, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "1", + "text": "OK", + "to": "99999999999999" + }, + { + "from": "0", + "text": "CRIT", + "to": "0" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "column": "value", + "condition": "", + "expr": "sum_over_time(nginx_connections_total{type=\"active\", namespace=\"openstack\"}[5m])", + "fill": "", + "format": "time_series", + "function": "last", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "groupByTags": [], + "groupby_field": "", + "interval": "", + "intervalFactor": 2, + "policy": "default", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "step": 120 + } + ], + "thresholds": "0,1", + "title": "Nginx", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "rgba(208, 53, 34, 0.82)", + "rgba(118, 245, 40, 0.73)", + "rgba(225, 177, 40, 0.59)" + ], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 2, + "x": 22, + "y": 1 + }, + "id": 13, + "interval": "> 60s", + "links": [ + { + "dashboard": "Memcached", + "name": "Drilldown dashboard", + "title": "Memcached", + "type": "dashboard" + } + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "column": "value", + "condition": "", + "expr": "min(memcached_up)", + "fill": "", + "format": "time_series", + "function": "last", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "groupByTags": [], + "groupby_field": "", + "interval": "", + "intervalFactor": 2, + "policy": "default", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "step": 120 + } + ], + "thresholds": "1,2", + "title": "Memcached", + "type": "singlestat", + "valueFontSize": "50%", + "valueMaps": [ + { + "op": "=", + "text": "no data", + "value": "null" + }, + { + "op": "=", + "text": "CRIT", + "value": "0" + }, + { + "op": "=", + "text": "OK", + "value": "1" + }, + { + "op": "=", + "text": "UNKW", + "value": "2" + } + ], + "valueName": "current" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 2, + "x": 22, + "y": 8 + }, + "id": 13, + "interval": "> 60s", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 3, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "free", + "column": "value", + "expr": "openstack_total_used_disk_GB{job=\"openstack-metrics\", region=\"$region\"} + openstack_total_free_disk_GB{job=\"openstack-metrics\", region=\"$region\"}", + "format": "time_series", + "function": "mean", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "0" + ], + "type": "fill" + } + ], + "groupByTags": [], + "intervalFactor": 2, + "policy": "default", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "step": 120 + }, + { + "alias": "used", + "column": "value", + "expr": "openstack_total_used_disk_GB{job=\"openstack-metrics\", region=\"$region\"}", + "format": "time_series", + "function": "mean", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "0" + ], + "type": "fill" + } + ], + "groupByTags": [], + "intervalFactor": 2, + "policy": "default", + "rawQuery": false, + "refId": "B", + "resultFormat": "time_series", + "step": 120 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Disk (used vs total)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "gbytes", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 15 + }, + "id": 29, + "panels": [], + "repeat": null, + "title": "Virtual resources", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 16 + }, + "id": 11, + "interval": "> 60s", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 3, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "free", + "column": "value", + "expr": "openstack_total_used_vcpus{job=\"openstack-metrics\", region=\"$region\"} + openstack_total_free_vcpus{job=\"openstack-metrics\", region=\"$region\"}", + "format": "time_series", + "function": "min", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "0" + ], + "type": "fill" + } + ], + "groupByTags": [], + "intervalFactor": 2, + "policy": "default", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "step": 120 + }, + { + "alias": "used", + "column": "value", + "expr": "openstack_total_used_vcpus{job=\"openstack-metrics\", region=\"$region\"}", + "format": "time_series", + "function": "max", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "0" + ], + "type": "fill" + } + ], + "groupByTags": [], + "intervalFactor": 2, + "policy": "default", + "rawQuery": false, + "refId": "B", + "resultFormat": "time_series", + "step": 120 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "VCPUs (total vs used)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 16 + }, + "id": 12, + "interval": "> 60s", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 3, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "free", + "column": "value", + "expr": "openstack_total_used_ram_MB{job=\"openstack-metrics\", region=\"$region\"} + openstack_total_free_ram_MB{job=\"openstack-metrics\", region=\"$region\"}", + "format": "time_series", + "function": "mean", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "0" + ], + "type": "fill" + } + ], + "groupByTags": [], + "intervalFactor": 2, + "policy": "default", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "step": 120 + }, + { + "alias": "used", + "column": "value", + "expr": "openstack_total_used_ram_MB{job=\"openstack-metrics\", region=\"$region\"}", + "format": "time_series", + "function": "mean", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "0" + ], + "type": "fill" + } + ], + "groupByTags": [], + "interval": "", + "intervalFactor": 2, + "policy": "default", + "rawQuery": false, + "refId": "B", + "resultFormat": "time_series", + "step": 120 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "RAM (total vs used)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "mbytes", + "label": "", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "dashes\"": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 23 + }, + "id": 27, + "interval": "> 60s", + "legend": { + "alignAsTable": false, + "avg": true, + "current": true, + "hideEmpty": true, + "hideZero": false, + "max": true, + "min": true, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 4, + "links": [], + "nullPointMode": null, + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "alias": "free", + "column": "value", + "expr": "sum(openstack_running_instances)", + "format": "time_series", + "function": "mean", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "0" + ], + "type": "fill" + } + ], + "groupByTags": [], + "interval": "15s", + "intervalFactor": 1, + "legendFormat": "{{ running_vms }}", + "policy": "default", + "rawQuery": false, + "refID": "A", + "refId": "A", + "resultFormat": "time_series" + }, + { + "alias": "used", + "column": "value", + "expr": "sum(openstack_total_running_instances)", + "format": "time_series", + "function": "mean", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "0" + ], + "type": "fill" + } + ], + "groupByTags": [], + "interval": "15s", + "intervalFactor": 1, + "legendFormat": "{{ total_vms }}", + "policy": "default", + "rawQuery": false, + "refID": "B", + "refId": "B", + "resultFormat": "time_series", + "step": 120 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "OpenStack Instances", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "transparent": true, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "5m", + "schemaVersion": 18, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "prometheus", + "value": "prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Prometheus datasource", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "", + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "region", + "options": [], + "query": "label_values(openstack_exporter_cache_refresh_duration_seconds, region)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "collapse": false, + "enable": true, + "notice": false, + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "status": "Stable", + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ], + "type": "timepicker" + }, + "timezone": "browser", + "title": "OpenStack Metrics", + "version": 1 + } + openstack-service: |- + { + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "4.5.2" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "singlestat", + "name": "Singlestat", + "version": "" + } + ], + "annotations": { + "enable": true, + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": false, + "overwrite": true, + "gnetId": null, + "graphTooltip": 1, + "id": 29, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 14, + "panels": [], + "repeat": null, + "title": "Service Status", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": true, + "colorValue": false, + "colors": [ + "rgba(225, 177, 40, 0.59)", + "rgba(200, 54, 35, 0.88)", + "rgba(118, 245, 40, 0.73)" + ], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 6, + "interval": "> 60s", + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "column": "value", + "condition": "", + "expr": "openstack_check_[[Service]]_api{job=\"openstack-metrics\",region=\"$region\"}", + "fill": "", + "format": "time_series", + "function": "last", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "null" + ], + "type": "fill" + } + ], + "groupByTags": [], + "groupby_field": "", + "interval": "", + "intervalFactor": 2, + "policy": "default", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "step": 120 + } + ], + "thresholds": "0,1", + "title": "", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "CRITICAL", + "value": "0" + }, + { + "op": "=", + "text": "OK", + "value": "1" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(200, 54, 35, 0.88)", + "rgba(118, 245, 40, 0.73)", + "rgba(225, 177, 40, 0.59)" + ], + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 7, + "w": 4, + "x": 4, + "y": 1 + }, + "id": 13, + "interval": "> 60s", + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "column": "value", + "condition": "", + "expr": "sum(nginx_responses_total{server_zone=~\"[[Service]].*\", status_code=\"5xx\",region=\"$region\"})", + "fill": "", + "format": "time_series", + "function": "count", + "groupBy": [ + { + "interval": "auto", + "params": [ + "auto" + ], + "type": "time" + }, + { + "params": [ + "0" + ], + "type": "fill" + } + ], + "groupby_field": "", + "interval": "", + "intervalFactor": 2, + "policy": "default", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "step": 120, + "tags": [] + } + ], + "thresholds": "", + "title": "HTTP 5xx errors", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "0", + "value": "null" + } + ], + "valueName": "current" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 16, + "x": 8, + "y": 1 + }, + "id": 7, + "interval": ">60s", + "legend": { + "alignAsTable": true, + "avg": true, + "current": false, + "max": true, + "min": true, + "show": true, + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(nginx_upstream_response_msecs_avg{upstream=~\"openstack-[[Service]].*\",region=\"$region\"}) by (upstream)", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 120 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "HTTP response time", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 8 + }, + "id": 9, + "interval": "> 60s", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": true, + "targets": [ + { + "alias": "healthy", + "column": "value", + "expr": "openstack_check_[[Service]]_api{region=\"$region\"}", + "format": "time_series", + "function": "last", + "groupBy": [ + { + "params": [ + "$interval" + ], + "type": "time" + }, + { + "params": [ + "0" + ], + "type": "fill" + } + ], + "groupByTags": [], + "intervalFactor": 2, + "policy": "default", + "rawQuery": false, + "refId": "A", + "resultFormat": "time_series", + "select": [], + "step": 120, + "tags": [] + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "API Availability", + "tooltip": { + "msResolution": false, + "shared": false, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": "", + "logBase": 1, + "max": 1, + "min": 0, + "show": false + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "{status_code=\"2xx\"}": "#629E51", + "{status_code=\"5xx\"}": "#BF1B00" + }, + "bars": true, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 0, + "grid": {}, + "gridPos": { + "h": 7, + "w": 16, + "x": 8, + "y": 8 + }, + "id": 8, + "interval": "> 60s", + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": false, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum(nginx_responses_total{server_zone=~\"[[Service]].*\",region=\"$region\"}) by (status_code)", + "format": "time_series", + "intervalFactor": 2, + "refId": "A", + "step": 120 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Number of HTTP responses", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + } + ], + "refresh": "5m", + "schemaVersion": 18, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "text": "prometheus", + "value": "prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Prometheus datasource", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "allValue": null, + "current": {}, + "datasource": "prometheus", + "definition": "", + "hide": 0, + "includeAll": false, + "label": "region", + "multi": false, + "name": "region", + "options": [], + "query": "label_values(openstack_exporter_cache_refresh_duration_seconds, region)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + "tags": [], + "text": "cinder", + "value": "cinder" + }, + "hide": 0, + "includeAll": false, + "label": null, + "multi": false, + "name": "Service", + "options": [ + { + "selected": false, + "text": "nova", + "value": "nova" + }, + { + "selected": false, + "text": "glance", + "value": "glance" + }, + { + "selected": false, + "text": "keystone", + "value": "keystone" + }, + { + "selected": true, + "text": "cinder", + "value": "cinder" + }, + { + "selected": false, + "text": "heat", + "value": "heat" + }, + { + "selected": false, + "text": "placement", + "value": "placement" + }, + { + "selected": false, + "text": "neutron", + "value": "neutron" + } + ], + "query": "nova,glance,keystone,cinder,heat,placement,neutron", + "skipUrlSync": false, + "type": "custom" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "collapse": false, + "enable": true, + "notice": false, + "now": true, + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "status": "Stable", + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ], + "type": "timepicker" + }, + "timezone": "browser", + "title": "Openstack Service", + "version": 1 + } +... diff --git a/values_overrides/grafana/persistentvolume.yaml b/values_overrides/grafana/persistentvolume.yaml new file mode 100644 index 0000000000..e07fde642a --- /dev/null +++ b/values_overrides/grafana/persistentvolume.yaml @@ -0,0 +1,554 @@ +# This overrides file provides a raw json file for a dashboard for +# the etcd +--- +conf: + dashboards: + openstack: + persistent_volume: |- + { + "__inputs": [ + { + "name": "prometheus", + "label": "Prometheus", + "description": "", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "5.0.0" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + } + ], + "annotations": { + "list": [ + ] + }, + "editable": false, + "overwrite": true, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [ + ], + "refresh": "", + "rows": [ + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + }, + "id": 2, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + ], + "spaceLength": 10, + "span": 9, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "(\n sum without(instance, node) (kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n -\n sum without(instance, node) (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n)\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Used Space", + "refId": "A" + }, + { + "expr": "sum without(instance, node) (kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Free Space", + "refId": "B" + } + ], + "thresholds": [ + ], + "timeFrom": null, + "timeShift": null, + "title": "Volume Space Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + ] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + }, + "id": 3, + "interval": null, + "links": [ + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "(\n kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n -\n kubelet_volume_stats_available_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n)\n/\nkubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n* 100\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "80, 90", + "title": "Volume Space Usage", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + }, + { + "collapse": false, + "collapsed": false, + "panels": [ + { + "aliasColors": { + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + }, + "id": 4, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": true, + "rightSide": false, + "show": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [ + ], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [ + ], + "spaceLength": 10, + "span": 9, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "sum without(instance, node) (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "Used inodes", + "refId": "A" + }, + { + "expr": "(\n sum without(instance, node) (kubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n -\n sum without(instance, node) (kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"})\n)\n", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": " Free inodes", + "refId": "B" + } + ], + "thresholds": [ + ], + "timeFrom": null, + "timeShift": null, + "title": "Volume inodes Usage", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + } + ] + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "rgba(50, 172, 45, 0.97)", + "rgba(237, 129, 40, 0.89)", + "rgba(245, 54, 54, 0.9)" + ], + "datasource": "$datasource", + "format": "percent", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": true, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + }, + "id": 5, + "interval": null, + "links": [ + ], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "span": 3, + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "kubelet_volume_stats_inodes_used{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n/\nkubelet_volume_stats_inodes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\", persistentvolumeclaim=\"$volume\"}\n* 100\n", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "", + "refId": "A" + } + ], + "thresholds": "80, 90", + "title": "Volume inodes Usage", + "tooltip": { + "shared": false + }, + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + } + ], + "repeat": null, + "repeatIteration": null, + "repeatRowId": null, + "showTitle": false, + "title": "Dashboard Row", + "titleSize": "h6", + "type": "row" + } + ], + "schemaVersion": 14, + "style": "dark", + "tags": [ + "kubernetes-mixin" + ], + "templating": { + "list": [ + { + "current": { + "text": "Prometheus", + "value": "Prometheus" + }, + "hide": 0, + "label": null, + "name": "datasource", + "options": [ + ], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "allValue": null, + "current": { + }, + "datasource": "$datasource", + "hide": 2, + "includeAll": false, + "label": "cluster", + "multi": false, + "name": "cluster", + "options": [ + ], + "query": "label_values(kubelet_volume_stats_capacity_bytes, cluster)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "Namespace", + "multi": false, + "name": "namespace", + "options": [ + ], + "query": "label_values(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\"}, namespace)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + ], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "allValue": null, + "current": { + }, + "datasource": "$datasource", + "hide": 0, + "includeAll": false, + "label": "PersistentVolumeClaim", + "multi": false, + "name": "volume", + "options": [ + ], + "query": "label_values(kubelet_volume_stats_capacity_bytes{cluster=\"$cluster\", job=\"kubelet\", namespace=\"$namespace\"}, persistentvolumeclaim)", + "refresh": 2, + "regex": "", + "sort": 1, + "tagValuesQuery": "", + "tags": [ + ], + "tagsQuery": "", + "type": "query", + "useTags": false + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "Persistent Volumes", + "version": 0 + } +... diff --git a/values_overrides/grafana/prometheus.yaml b/values_overrides/grafana/prometheus.yaml new file mode 100644 index 0000000000..46e2750f69 --- /dev/null +++ b/values_overrides/grafana/prometheus.yaml @@ -0,0 +1,3710 @@ +# NOTE(srwilkers): This overrides file provides a reference for a dashboard for +# Prometheus +--- +conf: + dashboards: + lma: + prometheus: |- + { + "__inputs": [ + { + "name": "DS_PROMETHEUS", + "label": "prometheus", + "description": "Prometheus which you want to monitor", + "type": "datasource", + "pluginId": "prometheus", + "pluginName": "Prometheus" + } + ], + "__requires": [ + { + "type": "grafana", + "id": "grafana", + "name": "Grafana", + "version": "4.6.0" + }, + { + "type": "panel", + "id": "graph", + "name": "Graph", + "version": "" + }, + { + "type": "datasource", + "id": "prometheus", + "name": "Prometheus", + "version": "1.0.0" + }, + { + "type": "panel", + "id": "singlestat", + "name": "Singlestat", + "version": "" + }, + { + "type": "panel", + "id": "text", + "name": "Text", + "version": "" + } + ], + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": "-- Grafana --", + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + }, + { + "datasource": "${DS_PROMETHEUS}", + "enable": true, + "expr": "count(sum(up{instance=\"$instance\"}) by (instance) < 1)", + "hide": false, + "iconColor": "rgb(250, 44, 18)", + "limit": 100, + "name": "downage", + "showIn": 0, + "step": "30s", + "tagKeys": "instance", + "textFormat": "prometheus down", + "titleFormat": "Downage", + "type": "alert" + }, + { + "datasource": "${DS_PROMETHEUS}", + "enable": true, + "expr": "sum(changes(prometheus_config_last_reload_success_timestamp_seconds[10m])) by (instance)", + "hide": false, + "iconColor": "#fceaca", + "limit": 100, + "name": "Reload", + "showIn": 0, + "step": "5m", + "tagKeys": "instance", + "tags": [], + "titleFormat": "Reload", + "type": "tags" + } + ] + }, + "description": "Dashboard for monitoring of Prometheus v2.x.x", + "overwrite": true, + "editable": false, + "gnetId": 3681, + "graphTooltip": 1, + "id": 41, + "links": [ + { + "icon": "info", + "tags": [], + "targetBlank": true, + "title": "Dashboard's Github ", + "tooltip": "Github repo of this dashboard", + "type": "link", + "url": "https://github.com/FUSAKLA/Prometheus2-grafana-dashboard" + }, + { + "icon": "doc", + "tags": [], + "targetBlank": true, + "title": "Prometheus Docs", + "tooltip": "", + "type": "link", + "url": "http://prometheus.io/docs/introduction/overview/" + } + ], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 53, + "panels": [], + "repeat": null, + "title": "Header instance info", + "type": "row" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#bf1b00" + ], + "datasource": "${DS_PROMETHEUS}", + "decimals": 1, + "format": "s", + "gauge": { + "maxValue": 1000000, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 0, + "y": 1 + }, + "id": 41, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "time() - process_start_time_seconds{instance=\"$instance\"}", + "format": "time_series", + "instant": false, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "", + "title": "Uptime", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#bf1b00" + ], + "datasource": "${DS_PROMETHEUS}", + "format": "short", + "gauge": { + "maxValue": 1000000, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 4, + "y": 1 + }, + "id": 42, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": true + }, + "tableColumn": "", + "targets": [ + { + "expr": "prometheus_tsdb_head_series{instance=\"$instance\"}", + "format": "time_series", + "instant": false, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "500000,800000,1000000", + "title": "Total count of time series", + "type": "singlestat", + "valueFontSize": "150%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "${DS_PROMETHEUS}", + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 12, + "y": 1 + }, + "id": 48, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "version", + "targets": [ + { + "expr": "prometheus_build_info{instance=\"$instance\"}", + "format": "table", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "", + "title": "Version", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": false, + "colors": [ + "#299c46", + "rgba(237, 129, 40, 0.89)", + "#d44a3a" + ], + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "format": "ms", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 4, + "x": 16, + "y": 1 + }, + "id": 49, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "prometheus_tsdb_head_max_time{instance=\"$instance\"} - prometheus_tsdb_head_min_time{instance=\"$instance\"}", + "format": "time_series", + "instant": true, + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "", + "title": "Actual head block length", + "type": "singlestat", + "valueFontSize": "80%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "current" + }, + { + "content": "", + "gridPos": { + "h": 5, + "w": 2, + "x": 20, + "y": 1 + }, + "height": "", + "id": 50, + "links": [], + "mode": "html", + "options": {}, + "title": "", + "transparent": true, + "type": "text" + }, + { + "cacheTimeout": null, + "colorBackground": false, + "colorValue": true, + "colors": [ + "#e6522c", + "rgba(237, 129, 40, 0.89)", + "#299c46" + ], + "datasource": "${DS_PROMETHEUS}", + "decimals": 1, + "format": "none", + "gauge": { + "maxValue": 100, + "minValue": 0, + "show": false, + "thresholdLabels": false, + "thresholdMarkers": true + }, + "gridPos": { + "h": 5, + "w": 2, + "x": 22, + "y": 1 + }, + "id": 52, + "interval": null, + "links": [], + "mappingType": 1, + "mappingTypes": [ + { + "name": "value to text", + "value": 1 + }, + { + "name": "range to text", + "value": 2 + } + ], + "maxDataPoints": 100, + "nullPointMode": "connected", + "nullText": null, + "options": {}, + "postfix": "", + "postfixFontSize": "50%", + "prefix": "", + "prefixFontSize": "50%", + "rangeMaps": [ + { + "from": "null", + "text": "N/A", + "to": "null" + } + ], + "sparkline": { + "fillColor": "rgba(31, 118, 189, 0.18)", + "full": false, + "lineColor": "rgb(31, 120, 193)", + "show": false + }, + "tableColumn": "", + "targets": [ + { + "expr": "2", + "format": "time_series", + "intervalFactor": 2, + "refId": "A" + } + ], + "thresholds": "10,20", + "title": "", + "transparent": true, + "type": "singlestat", + "valueFontSize": "200%", + "valueMaps": [ + { + "op": "=", + "text": "N/A", + "value": "null" + } + ], + "valueName": "avg" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 54, + "panels": [], + "repeat": null, + "title": "Main info", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 7 + }, + "id": 15, + "legend": { + "avg": true, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": true, + "steppedLine": false, + "targets": [ + { + "expr": "max(prometheus_engine_query_duration_seconds{instance=\"$instance\"}) by (instance, slice)", + "format": "time_series", + "intervalFactor": 1, + "legendFormat": "max duration for {{slice}}", + "metric": "prometheus_local_storage_rushed_mode", + "refId": "A", + "step": 900 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Query elapsed time", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 7 + }, + "id": 17, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(increase(prometheus_tsdb_head_series_created_total{instance=\"$instance\"}[$aggregation_interval])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "created on {{ instance }}", + "metric": "prometheus_local_storage_maintain_series_duration_seconds_count", + "refId": "A", + "step": 1800 + }, + { + "expr": "sum(increase(prometheus_tsdb_head_series_removed_total{instance=\"$instance\"}[$aggregation_interval])) by (instance) * -1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "removed on {{ instance }}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Head series created/deleted", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 7 + }, + "id": 13, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(increase(prometheus_target_scrapes_exceeded_sample_limit_total{instance=\"$instance\"}[$aggregation_interval])) by (instance) > 0", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "exceeded_sample_limit on {{ instance }}", + "metric": "prometheus_local_storage_chunk_ops_total", + "refId": "A", + "step": 1800 + }, + { + "expr": "sum(increase(prometheus_target_scrapes_sample_duplicate_timestamp_total{instance=\"$instance\"}[$aggregation_interval])) by (instance) > 0", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "duplicate_timestamp on {{ instance }}", + "metric": "prometheus_local_storage_chunk_ops_total", + "refId": "B", + "step": 1800 + }, + { + "expr": "sum(increase(prometheus_target_scrapes_sample_out_of_bounds_total{instance=\"$instance\"}[$aggregation_interval])) by (instance) > 0", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "out_of_bounds on {{ instance }}", + "metric": "prometheus_local_storage_chunk_ops_total", + "refId": "C", + "step": 1800 + }, + { + "expr": "sum(increase(prometheus_target_scrapes_sample_out_of_order_total{instance=\"$instance\"}[$aggregation_interval])) by (instance) > 0", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "out_of_order on {{ instance }}", + "metric": "prometheus_local_storage_chunk_ops_total", + "refId": "D", + "step": 1800 + }, + { + "expr": "sum(increase(prometheus_rule_evaluation_failures_total{instance=\"$instance\"}[$aggregation_interval])) by (instance) > 0", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "rule_evaluation_failure on {{ instance }}", + "metric": "prometheus_local_storage_chunk_ops_total", + "refId": "G", + "step": 1800 + }, + { + "expr": "sum(increase(prometheus_tsdb_compactions_failed_total{instance=\"$instance\"}[$aggregation_interval])) by (instance) > 0", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "tsdb_compactions_failed on {{ instance }}", + "metric": "prometheus_local_storage_chunk_ops_total", + "refId": "K", + "step": 1800 + }, + { + "expr": "sum(increase(prometheus_tsdb_reloads_failures_total{instance=\"$instance\"}[$aggregation_interval])) by (instance) > 0", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "tsdb_reloads_failures on {{ instance }}", + "metric": "prometheus_local_storage_chunk_ops_total", + "refId": "L", + "step": 1800 + }, + { + "expr": "sum(increase(prometheus_tsdb_head_series_not_found{instance=\"$instance\"}[$aggregation_interval])) by (instance) > 0", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "head_series_not_found on {{ instance }}", + "metric": "prometheus_local_storage_chunk_ops_total", + "refId": "E", + "step": 1800 + }, + { + "expr": "sum(increase(prometheus_evaluator_iterations_missed_total{instance=\"$instance\"}[$aggregation_interval])) by (instance) > 0", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "evaluator_iterations_missed on {{ instance }}", + "metric": "prometheus_local_storage_chunk_ops_total", + "refId": "O", + "step": 1800 + }, + { + "expr": "sum(increase(prometheus_evaluator_iterations_skipped_total{instance=\"$instance\"}[$aggregation_interval])) by (instance) > 0", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "evaluator_iterations_skipped on {{ instance }}", + "metric": "prometheus_local_storage_chunk_ops_total", + "refId": "P", + "step": 1800 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Prometheus errors", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 55, + "panels": [], + "repeat": null, + "title": "Scrape & rule duration", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "description": "", + "editable": true, + "error": false, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 25, + "legend": { + "alignAsTable": true, + "avg": true, + "current": true, + "max": true, + "min": false, + "show": false, + "sort": "max", + "sortDesc": true, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 2, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_target_interval_length_seconds{instance=\"$instance\",quantile=\"0.99\"} - 60", + "format": "time_series", + "interval": "2m", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "metric": "", + "refId": "A", + "step": 300 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Scrape delay (counts with 1m scrape interval)", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 15 + }, + "id": 14, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "Queue length", + "yaxis": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(prometheus_evaluator_duration_seconds{instance=\"$instance\"}) by (instance, quantile)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Queue length", + "metric": "prometheus_local_storage_indexing_queue_length", + "refId": "B", + "step": 1800 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Rule evaulation duration", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 22 + }, + "id": 56, + "panels": [], + "repeat": null, + "title": "Requests & queries", + "type": "row" + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 23 + }, + "id": 18, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(increase(http_requests_total{instance=\"$instance\"}[$aggregation_interval])) by (instance, handler) > 0", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ handler }} on {{ instance }}", + "metric": "", + "refId": "A", + "step": 1800 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Request count", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 23 + }, + "id": 16, + "legend": { + "avg": false, + "current": false, + "hideEmpty": true, + "hideZero": true, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max(sum(http_request_duration_microseconds{instance=\"$instance\"}) by (instance, handler, quantile)) by (instance, handler) > 0", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{ handler }} on {{ instance }}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Request duration per handler", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "µs", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 23 + }, + "id": 19, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(increase(http_request_size_bytes{instance=\"$instance\", quantile=\"0.99\"}[$aggregation_interval])) by (instance, handler) > 0", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "{{ handler }} in {{ instance }}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Request size by handler", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Allocated bytes": "#F9BA8F", + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max count collector": "#bf1b00", + "Max count harvester": "#bf1b00", + "Max to persist": "#3F6833", + "RSS": "#890F02" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 23 + }, + "id": 8, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/Max.*/", + "fill": 0, + "linewidth": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(prometheus_engine_queries{instance=\"$instance\"}) by (instance, handler)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Current count ", + "metric": "last", + "refId": "A", + "step": 1800 + }, + { + "expr": "sum(prometheus_engine_queries_concurrent_max{instance=\"$instance\"}) by (instance, handler)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Max count", + "metric": "last", + "refId": "B", + "step": 1800 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Cont of concurent queries", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 57, + "panels": [], + "repeat": null, + "title": "Alerting", + "type": "row" + }, + { + "aliasColors": { + "Alert queue capacity on o collector": "#bf1b00", + "Alert queue capacity on o harvester": "#bf1b00", + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 31 + }, + "id": 20, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/.*capacity.*/", + "fill": 0, + "linewidth": 2 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(prometheus_notifications_queue_capacity{instance=\"$instance\"})by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Alert queue capacity ", + "metric": "prometheus_local_storage_checkpoint_last_size_bytes", + "refId": "A", + "step": 1800 + }, + { + "expr": "sum(prometheus_notifications_queue_length{instance=\"$instance\"})by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Alert queue size on ", + "metric": "prometheus_local_storage_checkpoint_last_size_bytes", + "refId": "B", + "step": 1800 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Alert queue size", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 31 + }, + "id": 21, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(prometheus_notifications_alertmanagers_discovered{instance=\"$instance\"}) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Checkpoint chunks written/s", + "metric": "prometheus_local_storage_checkpoint_series_chunks_written_sum", + "refId": "A", + "step": 1800 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Count of discovered alertmanagers", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 31 + }, + "id": 39, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(increase(prometheus_notifications_dropped_total{instance=\"$instance\"}[$aggregation_interval])) by (instance) > 0", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "notifications_dropped on {{ instance }}", + "metric": "prometheus_local_storage_chunk_ops_total", + "refId": "F", + "step": 1800 + }, + { + "expr": "sum(increase(prometheus_rule_evaluation_failures_total{rule_type=\"alerting\",instance=\"$instance\"}[$aggregation_interval])) by (rule_type,instance) > 0", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "rule_evaluation_failures on {{ instance }}", + "metric": "prometheus_local_storage_chunk_ops_total", + "refId": "A", + "step": 1800 + } + ], + "thresholds": [], + "timeFrom": null, + "timeRegions": [], + "timeShift": null, + "title": "Alerting errors", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + } + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 38 + }, + "id": 58, + "panels": [], + "repeat": null, + "title": "Service discovery", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 39 + }, + "id": 45, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "increase(prometheus_target_sync_length_seconds_count{scrape_job=\"kubernetes-service-endpoints\"}[$aggregation_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Count of target synces", + "refId": "A", + "step": 240 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Kubernetes SD sync count", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 39 + }, + "id": 46, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(increase(prometheus_target_scrapes_exceeded_sample_limit_total{instance=\"$instance\"}[$aggregation_interval])) by (instance) > 0", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "exceeded_sample_limit on {{ instance }}", + "metric": "prometheus_local_storage_chunk_ops_total", + "refId": "A", + "step": 1800 + }, + { + "expr": "sum(increase(prometheus_sd_file_read_errors_total{instance=\"$instance\"}[$aggregation_interval])) by (instance) > 0", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "sd_file_read_error on {{ instance }}", + "metric": "prometheus_local_storage_chunk_ops_total", + "refId": "E", + "step": 1800 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Service discovery errors", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 46 + }, + "id": 59, + "panels": [], + "repeat": null, + "title": "TSDB stats", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 47 + }, + "id": 36, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(increase(prometheus_tsdb_reloads_total{instance=\"$instance\"}[30m])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ instance }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Reloaded block from disk", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 47 + }, + "id": 5, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(prometheus_tsdb_blocks_loaded{instance=\"$instance\"}) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Loaded data blocks", + "metric": "prometheus_local_storage_memory_chunkdescs", + "refId": "A", + "step": 1800 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Loaded data blocks", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 47 + }, + "id": 3, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "prometheus_tsdb_head_series{instance=\"$instance\"}", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Time series count", + "metric": "prometheus_local_storage_memory_series", + "refId": "A", + "step": 1800 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Time series total count", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 47 + }, + "id": 1, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(prometheus_tsdb_head_samples_appended_total{instance=\"$instance\"}[$aggregation_interval])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "samples/s {{instance}}", + "metric": "prometheus_local_storage_ingested_samples_total", + "refId": "A", + "step": 1800 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Samples Appended per second", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": "", + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 54 + }, + "id": 60, + "panels": [], + "repeat": null, + "title": "Head block stats", + "type": "row" + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833", + "To persist": "#9AC48A" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 55 + }, + "id": 2, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/Max.*/", + "fill": 0 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(prometheus_tsdb_head_chunks{instance=\"$instance\"}) by (instance)", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Head chunk count", + "metric": "prometheus_local_storage_memory_chunks", + "refId": "A", + "step": 1800 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Head chunks count", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 55 + }, + "id": 35, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "max(prometheus_tsdb_head_max_time{instance=\"$instance\"}) by (instance) - min(prometheus_tsdb_head_min_time{instance=\"$instance\"}) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ instance }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Length of head block", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 55 + }, + "id": 4, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(prometheus_tsdb_head_chunks_created_total{instance=\"$instance\"}[$aggregation_interval])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "created on {{ instance }}", + "refId": "B" + }, + { + "expr": "sum(rate(prometheus_tsdb_head_chunks_removed_total{instance=\"$instance\"}[$aggregation_interval])) by (instance) * -1", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "deleted on {{ instance }}", + "refId": "C" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Head Chunks Created/Deleted per second", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 62 + }, + "id": 61, + "panels": [], + "repeat": null, + "title": "Data maintenance", + "type": "row" + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 0, + "y": 63 + }, + "id": 33, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(increase(prometheus_tsdb_compaction_duration_sum{instance=\"$instance\"}[30m]) / increase(prometheus_tsdb_compaction_duration_count{instance=\"$instance\"}[30m])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ instance }}", + "refId": "B" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Compaction duration", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 6, + "y": 63 + }, + "id": 34, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(prometheus_tsdb_head_gc_duration_seconds{instance=\"$instance\"}) by (instance, quantile)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ quantile }} on {{ instance }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Go Garbage collection duration", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 12, + "y": 63 + }, + "id": 37, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(prometheus_tsdb_wal_truncate_duration_seconds{instance=\"$instance\"}) by (instance, quantile)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ quantile }} on {{ instance }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "WAL truncate duration seconds", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "fill": 1, + "gridPos": { + "h": 7, + "w": 6, + "x": 18, + "y": 63 + }, + "id": 38, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "connected", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(tsdb_wal_fsync_duration_seconds{instance=\"$instance\"}) by (instance, quantile)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "{{ quantile }} {{ instance }}", + "refId": "A" + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "WAL fsync duration seconds", + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "s", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 70 + }, + "id": 62, + "panels": [], + "repeat": null, + "title": "RAM&CPU", + "type": "row" + }, + { + "aliasColors": { + "Allocated bytes": "#7EB26D", + "Allocated bytes - 1m max": "#BF1B00", + "Allocated bytes - 1m min": "#BF1B00", + "Allocated bytes - 5m max": "#BF1B00", + "Allocated bytes - 5m min": "#BF1B00", + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833", + "RSS": "#447EBC" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": null, + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 71 + }, + "id": 6, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [ + { + "alias": "/-/", + "fill": 0 + }, + { + "alias": "collector heap size", + "color": "#E0752D", + "fill": 0, + "linewidth": 2 + }, + { + "alias": "collector kubernetes memory limit", + "color": "#BF1B00", + "fill": 0, + "linewidth": 3 + } + ], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(process_resident_memory_bytes{instance=\"$instance\"}) by (instance)", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Total resident memory - {{instance}}", + "metric": "process_resident_memory_bytes", + "refId": "B", + "step": 1800 + }, + { + "expr": "sum(go_memstats_alloc_bytes{instance=\"$instance\"}) by (instance)", + "format": "time_series", + "hide": false, + "intervalFactor": 2, + "legendFormat": "Total llocated bytes - {{instance}}", + "metric": "go_memstats_alloc_bytes", + "refId": "A", + "step": 1800 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Memory", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": { + "Allocated bytes": "#F9BA8F", + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833", + "RSS": "#890F02" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 8, + "y": 71 + }, + "id": 7, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "rate(go_memstats_alloc_bytes_total{instance=\"$instance\"}[$aggregation_interval])", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "Allocated Bytes/s", + "metric": "go_memstats_alloc_bytes", + "refId": "A", + "step": 1800 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Allocations per second", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "bytes", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "decimals": 2, + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 8, + "x": 16, + "y": 71 + }, + "id": 9, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "hideEmpty": false, + "max": false, + "min": false, + "rightSide": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(rate(process_cpu_seconds_total{instance=\"$instance\"}[$aggregation_interval])) by (instance)", + "format": "time_series", + "intervalFactor": 2, + "legendFormat": "CPU/s", + "metric": "prometheus_local_storage_ingested_samples_total", + "refId": "B", + "step": 1800 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "CPU per second", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [ + "avg" + ] + }, + "yaxes": [ + { + "format": "none", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 78 + }, + "id": 63, + "panels": [], + "repeat": null, + "title": "Contrac errors", + "type": "row" + }, + { + "aliasColors": { + "Chunks": "#1F78C1", + "Chunks to persist": "#508642", + "Max chunks": "#052B51", + "Max to persist": "#3F6833" + }, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "${DS_PROMETHEUS}", + "editable": true, + "error": false, + "fill": 1, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 79 + }, + "id": 47, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": false, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "options": {}, + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "sum(increase(net_conntrack_dialer_conn_failed_total{instance=\"$instance\"}[$aggregation_interval])) by (instance) > 0", + "format": "time_series", + "hide": false, + "interval": "", + "intervalFactor": 2, + "legendFormat": "conntrack_dialer_conn_failed on {{ instance }}", + "metric": "prometheus_local_storage_chunk_ops_total", + "refId": "M", + "step": 1800 + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Net errors", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": "0", + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "refresh": "5m", + "schemaVersion": 18, + "style": "dark", + "tags": [ + "prometheus" + ], + "templating": { + "list": [ + { + "auto": true, + "auto_count": 30, + "auto_min": "2m", + "current": { + "text": "auto", + "value": "$__auto_interval_aggregation_interval" + }, + "hide": 0, + "label": "aggregation intarval", + "name": "aggregation_interval", + "options": [ + { + "selected": true, + "text": "auto", + "value": "$__auto_interval_aggregation_interval" + }, + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "1m,10m,30m,1h,6h,12h,1d,7d,14d,30d", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + }, + { + "allValue": null, + "current": {}, + "datasource": "${DS_PROMETHEUS}", + "definition": "", + "hide": 0, + "includeAll": false, + "label": "Instance", + "multi": false, + "name": "instance", + "options": [], + "query": "label_values(prometheus_build_info, instance)", + "refresh": 2, + "regex": "", + "skipUrlSync": false, + "sort": 2, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, + { + "current": { + "text": "prometheus", + "value": "prometheus" + }, + "hide": 0, + "includeAll": false, + "label": "Prometheus datasource", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "text": "No data sources found", + "value": "" + }, + "hide": 0, + "includeAll": false, + "label": "InfluxDB datasource", + "multi": false, + "name": "influx_datasource", + "options": [], + "query": "influxdb", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "browser", + "title": "Prometheus2.0 (v1.0.0 by FUSAKLA)", + "version": 1 + } +... diff --git a/values_overrides/grafana/sqlite3.yaml b/values_overrides/grafana/sqlite3.yaml new file mode 100644 index 0000000000..3bc2ad704c --- /dev/null +++ b/values_overrides/grafana/sqlite3.yaml @@ -0,0 +1,24 @@ +--- +dependencies: + static: + grafana: + jobs: null + services: null +manifests: + job_db_init: false + job_db_init_session: false + job_db_session_sync: false + job_image_repo_sync: true + job_run_migrator: false + job_set_admin_user: false + secret_db: false + secret_db_session: false +conf: + grafana: + database: + type: sqlite3 + path: /var/lib/grafana/data/sqlite3.db + session: + provider: file + provider_config: sessions +... diff --git a/values_overrides/grafana/tls.yaml b/values_overrides/grafana/tls.yaml new file mode 100644 index 0000000000..19c09c9930 --- /dev/null +++ b/values_overrides/grafana/tls.yaml @@ -0,0 +1,39 @@ +--- +conf: + grafana: + database: + ssl_mode: true + ca_cert_path: /etc/mysql/certs/ca.crt + client_key_path: /etc/mysql/certs/tls.key + client_cert_path: /etc/mysql/certs/tls.crt + provisioning: + datasources: + template: | + {{ $prom_host := tuple "monitoring" "internal" . | include "helm-toolkit.endpoints.hostname_fqdn_endpoint_lookup" }} + {{ $prom_uri := printf "https://%s" $prom_host }} + apiVersion: 1 + datasources: + - name: prometheus + type: prometheus + access: proxy + orgId: 1 + editable: true + basicAuth: true + basicAuthUser: {{ .Values.endpoints.monitoring.auth.user.username }} + jsonData: + tlsAuthWithCACert: true + secureJsonData: + basicAuthPassword: {{ .Values.endpoints.monitoring.auth.user.password }} + tlsCACert: $CACERT + url: {{ $prom_uri }} +endpoints: + grafana: + host_fqdn_override: + default: + tls: + issuerRef: + name: ca-issuer + kind: ClusterIssuer +manifests: + certificates: true +... diff --git a/values_overrides/kibana/2023.1-ubuntu_focal.yaml b/values_overrides/kibana/2023.1-ubuntu_focal.yaml new file mode 100644 index 0000000000..2a5286d2ff --- /dev/null +++ b/values_overrides/kibana/2023.1-ubuntu_focal.yaml @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + register_kibana_indexes: docker.io/openstackhelm/heat:2023.1-ubuntu_focal + flush_kibana_metadata: docker.io/openstackhelm/heat:2023.1-ubuntu_focal +... diff --git a/values_overrides/kibana/2024.1-ubuntu_jammy.yaml b/values_overrides/kibana/2024.1-ubuntu_jammy.yaml new file mode 100644 index 0000000000..da943640df --- /dev/null +++ b/values_overrides/kibana/2024.1-ubuntu_jammy.yaml @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + register_kibana_indexes: docker.io/openstackhelm/heat:2024.1-ubuntu_jammy + flush_kibana_metadata: docker.io/openstackhelm/heat:2024.1-ubuntu_jammy +... diff --git a/values_overrides/kibana/2024.2-ubuntu_jammy.yaml b/values_overrides/kibana/2024.2-ubuntu_jammy.yaml new file mode 100644 index 0000000000..ba76768c28 --- /dev/null +++ b/values_overrides/kibana/2024.2-ubuntu_jammy.yaml @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + register_kibana_indexes: docker.io/openstackhelm/heat:2024.2-ubuntu_jammy + flush_kibana_metadata: docker.io/openstackhelm/heat:2024.2-ubuntu_jammy +... diff --git a/values_overrides/kibana/apparmor.yaml b/values_overrides/kibana/apparmor.yaml new file mode 100644 index 0000000000..271646cc0f --- /dev/null +++ b/values_overrides/kibana/apparmor.yaml @@ -0,0 +1,15 @@ +--- +pod: + mandatory_access_control: + type: apparmor + kibana: + kibana: runtime/default + init: runtime/default + apache-proxy: runtime/default + register-kibana-indexes: + register-kibana-indexes: runtime/default + init: runtime/default + flush-kibana-metadata: + flush-kibana-metadata: runtime/default + init: runtime/default +... diff --git a/values_overrides/kibana/tls.yaml b/values_overrides/kibana/tls.yaml new file mode 100644 index 0000000000..f40c2eea11 --- /dev/null +++ b/values_overrides/kibana/tls.yaml @@ -0,0 +1,24 @@ +--- +conf: + kibana: + elasticsearch: + ssl: + certificateAuthorities: ["/etc/elasticsearch/certs/ca.crt"] + verificationMode: certificate +endpoints: + elasticsearch: + scheme: + default: "https" + port: + http: + default: 443 + kibana: + host_fqdn_override: + default: + tls: + issuerRef: + name: ca-issuer + kind: ClusterIssue +manifests: + certificates: true +... diff --git a/values_overrides/kubernetes-keystone-webhook/2023.1-ubuntu_focal.yaml b/values_overrides/kubernetes-keystone-webhook/2023.1-ubuntu_focal.yaml new file mode 100644 index 0000000000..11f1b479d4 --- /dev/null +++ b/values_overrides/kubernetes-keystone-webhook/2023.1-ubuntu_focal.yaml @@ -0,0 +1,17 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + scripted_test: docker.io/openstackhelm/heat:2023.1-ubuntu_focal +... diff --git a/values_overrides/kubernetes-keystone-webhook/2024.1-ubuntu_jammy.yaml b/values_overrides/kubernetes-keystone-webhook/2024.1-ubuntu_jammy.yaml new file mode 100644 index 0000000000..4b5f3b6075 --- /dev/null +++ b/values_overrides/kubernetes-keystone-webhook/2024.1-ubuntu_jammy.yaml @@ -0,0 +1,17 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + scripted_test: docker.io/openstackhelm/heat:2024.1-ubuntu_jammy +... diff --git a/values_overrides/kubernetes-keystone-webhook/2024.2-ubuntu_jammy.yaml b/values_overrides/kubernetes-keystone-webhook/2024.2-ubuntu_jammy.yaml new file mode 100644 index 0000000000..05b1d25a46 --- /dev/null +++ b/values_overrides/kubernetes-keystone-webhook/2024.2-ubuntu_jammy.yaml @@ -0,0 +1,17 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + scripted_test: docker.io/openstackhelm/heat:2024.2-ubuntu_jammy +... diff --git a/values_overrides/kubernetes-node-problem-detector/apparmor.yaml b/values_overrides/kubernetes-node-problem-detector/apparmor.yaml new file mode 100644 index 0000000000..7f4076991b --- /dev/null +++ b/values_overrides/kubernetes-node-problem-detector/apparmor.yaml @@ -0,0 +1,8 @@ +--- +pod: + mandatory_access_control: + type: apparmor + node-problem-detector: + node-problem-detector: runtime/default + init: runtime/default +... diff --git a/values_overrides/libvirt/2023.1-ubuntu_focal.yaml b/values_overrides/libvirt/2023.1-ubuntu_focal.yaml new file mode 100644 index 0000000000..950476dbec --- /dev/null +++ b/values_overrides/libvirt/2023.1-ubuntu_focal.yaml @@ -0,0 +1,5 @@ +--- +images: + tags: + libvirt: docker.io/openstackhelm/libvirt:latest-ubuntu_focal +... diff --git a/values_overrides/libvirt/2023.1-ubuntu_jammy.yaml b/values_overrides/libvirt/2023.1-ubuntu_jammy.yaml new file mode 100644 index 0000000000..fb478472c7 --- /dev/null +++ b/values_overrides/libvirt/2023.1-ubuntu_jammy.yaml @@ -0,0 +1,5 @@ +--- +images: + tags: + libvirt: docker.io/openstackhelm/libvirt:2023.1-ubuntu_jammy +... diff --git a/values_overrides/libvirt/2023.2-ubuntu_jammy.yaml b/values_overrides/libvirt/2023.2-ubuntu_jammy.yaml new file mode 100644 index 0000000000..e4c1ef7606 --- /dev/null +++ b/values_overrides/libvirt/2023.2-ubuntu_jammy.yaml @@ -0,0 +1,5 @@ +--- +images: + tags: + libvirt: docker.io/openstackhelm/libvirt:2023.2-ubuntu_jammy +... diff --git a/values_overrides/libvirt/2024.1-ubuntu_jammy.yaml b/values_overrides/libvirt/2024.1-ubuntu_jammy.yaml new file mode 100644 index 0000000000..4474d82216 --- /dev/null +++ b/values_overrides/libvirt/2024.1-ubuntu_jammy.yaml @@ -0,0 +1,5 @@ +--- +images: + tags: + libvirt: docker.io/openstackhelm/libvirt:2024.1-ubuntu_jammy +... diff --git a/values_overrides/libvirt/2024.2-ubuntu_jammy.yaml b/values_overrides/libvirt/2024.2-ubuntu_jammy.yaml new file mode 100644 index 0000000000..4474d82216 --- /dev/null +++ b/values_overrides/libvirt/2024.2-ubuntu_jammy.yaml @@ -0,0 +1,5 @@ +--- +images: + tags: + libvirt: docker.io/openstackhelm/libvirt:2024.1-ubuntu_jammy +... diff --git a/values_overrides/libvirt/apparmor.yaml b/values_overrides/libvirt/apparmor.yaml new file mode 100644 index 0000000000..3990314303 --- /dev/null +++ b/values_overrides/libvirt/apparmor.yaml @@ -0,0 +1,7 @@ +--- +pod: + mandatory_access_control: + type: apparmor + libvirt-libvirt-default: + libvirt: runtime/default +... diff --git a/values_overrides/libvirt/cinder-external-ceph-backend.yaml b/values_overrides/libvirt/cinder-external-ceph-backend.yaml new file mode 100644 index 0000000000..fe1c7889f8 --- /dev/null +++ b/values_overrides/libvirt/cinder-external-ceph-backend.yaml @@ -0,0 +1,16 @@ +# Note: This yaml file serves as an example for overriding the manifest +# to enable additional externally managed Ceph Cinder backend. When additional +# externally managed Ceph Cinder backend is provisioned as shown in +# cinder/values_overrides/external-ceph-backend.yaml of repo openstack-helm, +# below override is needed to store the secret key of the cinder user in +# libvirt. +--- +conf: + ceph: + cinder: + external_ceph: + enabled: true + user: cinder2 + secret_uuid: 3f0133e4-8384-4743-9473-fecacc095c74 + user_secret_name: cinder-volume-external-rbd-keyring +... diff --git a/values_overrides/libvirt/netpol.yaml b/values_overrides/libvirt/netpol.yaml new file mode 100644 index 0000000000..7eedf73caf --- /dev/null +++ b/values_overrides/libvirt/netpol.yaml @@ -0,0 +1,4 @@ +--- +manifests: + network_policy: true +... diff --git a/values_overrides/libvirt/node_overrides.yaml b/values_overrides/libvirt/node_overrides.yaml new file mode 100644 index 0000000000..1464fec522 --- /dev/null +++ b/values_overrides/libvirt/node_overrides.yaml @@ -0,0 +1,21 @@ +--- +# We have two nodes labeled with node-nics-type=4nics and node-nics-type=2nics +# on first node we pick up libvirt bind address from ens3 interface +# on second node we pick up libvirt bind address from ens0 interface +overrides: + libvirt_libvirt: + overrides_default: false + labels: + node-nics-type::4nics: + values: + conf: + dynamic_options: + libvirt: + listen_interface: ens3 + node-nics-type::2nics: + values: + conf: + dynamic_options: + libvirt: + listen_interface: ens0 +... diff --git a/values_overrides/libvirt/ovn.yaml b/values_overrides/libvirt/ovn.yaml new file mode 100644 index 0000000000..b95798f358 --- /dev/null +++ b/values_overrides/libvirt/ovn.yaml @@ -0,0 +1,8 @@ +--- +dependencies: + dynamic: + targeted: + openvswitch: + libvirt: + pod: [] +... diff --git a/values_overrides/libvirt/ssl.yaml b/values_overrides/libvirt/ssl.yaml new file mode 100644 index 0000000000..1cebd56f4b --- /dev/null +++ b/values_overrides/libvirt/ssl.yaml @@ -0,0 +1,7 @@ +--- +conf: + libvirt: + listen_tcp: "0" + listen_tls: "1" + listen_addr: 0.0.0.0 +... diff --git a/values_overrides/local-storage/local-storage.yaml b/values_overrides/local-storage/local-storage.yaml new file mode 100644 index 0000000000..f3267f02c0 --- /dev/null +++ b/values_overrides/local-storage/local-storage.yaml @@ -0,0 +1,37 @@ +--- +conf: + persistent_volumes: + - name: local-persistent-volume-0 + reclaim_policy: Delete + storage_capacity: "1Gi" + access_modes: ["ReadWriteOnce"] + local_path: /srv/local-volume-0 + - name: local-persistent-volume-1 + reclaim_policy: Delete + storage_capacity: "1Gi" + access_modes: ["ReadWriteOnce"] + local_path: /srv/local-volume-1 + - name: local-persistent-volume-2 + reclaim_policy: Delete + storage_capacity: "1Gi" + access_modes: ["ReadWriteOnce"] + local_path: /srv/local-volume-2 + - name: local-persistent-volume-3 + reclaim_policy: Delete + storage_capacity: "1Gi" + access_modes: ["ReadWriteOnce"] + local_path: /srv/local-volume-3 + - name: local-persistent-volume-4 + reclaim_policy: Delete + storage_capacity: "1Gi" + access_modes: ["ReadWriteOnce"] + local_path: /srv/local-volume-4 + - name: local-persistent-volume-5 + reclaim_policy: Delete + storage_capacity: "1Gi" + access_modes: ["ReadWriteOnce"] + local_path: /srv/local-volume-5 +manifests: + storage_class: true + persistent_volumes: true +... diff --git a/values_overrides/mariadb-backup/2023.1-ubuntu_focal.yaml b/values_overrides/mariadb-backup/2023.1-ubuntu_focal.yaml new file mode 100644 index 0000000000..4c9e14eccb --- /dev/null +++ b/values_overrides/mariadb-backup/2023.1-ubuntu_focal.yaml @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + prometheus_mysql_exporter_helm_tests: docker.io/openstackhelm/heat:2023.1-ubuntu_focal + ks_user: docker.io/openstackhelm/heat:2023.1-ubuntu_focal +... diff --git a/values_overrides/mariadb-backup/2023.2-ubuntu_jammy.yaml b/values_overrides/mariadb-backup/2023.2-ubuntu_jammy.yaml new file mode 100644 index 0000000000..e234a9e0aa --- /dev/null +++ b/values_overrides/mariadb-backup/2023.2-ubuntu_jammy.yaml @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + prometheus_mysql_exporter_helm_tests: docker.io/openstackhelm/heat:2023.2-ubuntu_jammy + ks_user: docker.io/openstackhelm/heat:2023.2-ubuntu_jammy +... diff --git a/values_overrides/mariadb-backup/2024.1-ubuntu_jammy.yaml b/values_overrides/mariadb-backup/2024.1-ubuntu_jammy.yaml new file mode 100644 index 0000000000..6c87b70789 --- /dev/null +++ b/values_overrides/mariadb-backup/2024.1-ubuntu_jammy.yaml @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + prometheus_mysql_exporter_helm_tests: docker.io/openstackhelm/heat:2024.1-ubuntu_jammy + ks_user: docker.io/openstackhelm/heat:2024.1-ubuntu_jammy +... diff --git a/values_overrides/mariadb-backup/2024.2-ubuntu_jammy.yaml b/values_overrides/mariadb-backup/2024.2-ubuntu_jammy.yaml new file mode 100644 index 0000000000..78d19b0003 --- /dev/null +++ b/values_overrides/mariadb-backup/2024.2-ubuntu_jammy.yaml @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + prometheus_mysql_exporter_helm_tests: docker.io/openstackhelm/heat:2024.2-ubuntu_jammy + ks_user: docker.io/openstackhelm/heat:2024.2-ubuntu_jammy +... diff --git a/values_overrides/mariadb-backup/apparmor.yaml b/values_overrides/mariadb-backup/apparmor.yaml new file mode 100644 index 0000000000..fa458fa556 --- /dev/null +++ b/values_overrides/mariadb-backup/apparmor.yaml @@ -0,0 +1,15 @@ +--- +pod: + mandatory_access_control: + type: apparmor + mariadb-backup: + init: runtime/default + mariadb-backup: runtime/default + mariadb-verify-server: runtime/default + create-sql-user: + init: runtime/default + exporter-create-sql-user: runtime/default + +manifests: + cron_job_mariadb_backup: true +... diff --git a/values_overrides/mariadb-backup/backups.yaml b/values_overrides/mariadb-backup/backups.yaml new file mode 100644 index 0000000000..5a7de206c1 --- /dev/null +++ b/values_overrides/mariadb-backup/backups.yaml @@ -0,0 +1,15 @@ +--- +conf: + backup: + enabled: true + remote_backup: + enabled: false +volume: + backup: + enabled: true +manifests: + pvc_backup: true + job_ks_user: false + cron_job_mariadb_backup: true + secret_backup_restore: true +... diff --git a/values_overrides/mariadb-backup/staggered-backups.yaml b/values_overrides/mariadb-backup/staggered-backups.yaml new file mode 100644 index 0000000000..03412d748c --- /dev/null +++ b/values_overrides/mariadb-backup/staggered-backups.yaml @@ -0,0 +1,38 @@ +--- +conf: + backup: + enabled: true + remote_backup: + enabled: false +pod: + labels: + backup: + staggered_backups: enabled + affinity: + mariadb_backup: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: status.phase + operator: NotIn + values: + - Running + - key: staggered-backups + operator: In + values: + - enabled + namespaces: + - openstack + - osh-infra + - ucp + topologyKey: kubernetes.io/os +volume: + backup: + enabled: true +manifests: + pvc_backup: true + job_ks_user: false + cron_job_mariadb_backup: true + secret_backup_restore: true +... diff --git a/values_overrides/mariadb-backup/tls.yaml b/values_overrides/mariadb-backup/tls.yaml new file mode 100644 index 0000000000..d50f732bfd --- /dev/null +++ b/values_overrides/mariadb-backup/tls.yaml @@ -0,0 +1,13 @@ +--- +endpoints: + oslo_db: + host_fqdn_override: + default: + tls: + secretName: mariadb-tls-direct + issuerRef: + name: ca-issuer + kind: ClusterIssuer +manifests: + certificates: true +... diff --git a/values_overrides/mariadb-backup/ubuntu_focal.yaml b/values_overrides/mariadb-backup/ubuntu_focal.yaml new file mode 100644 index 0000000000..0a2b327753 --- /dev/null +++ b/values_overrides/mariadb-backup/ubuntu_focal.yaml @@ -0,0 +1,19 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + mariadb: docker.io/openstackhelm/mariadb:latest-ubuntu_focal + ks_user: docker.io/openstackhelm/heat:wallaby-ubuntu_focal + mariadb_backup: quay.io/airshipit/porthole-mysqlclient-utility:latest-ubuntu_focal +... diff --git a/values_overrides/mariadb-cluster/2023.1-ubuntu_focal.yaml b/values_overrides/mariadb-cluster/2023.1-ubuntu_focal.yaml new file mode 100644 index 0000000000..4c9e14eccb --- /dev/null +++ b/values_overrides/mariadb-cluster/2023.1-ubuntu_focal.yaml @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + prometheus_mysql_exporter_helm_tests: docker.io/openstackhelm/heat:2023.1-ubuntu_focal + ks_user: docker.io/openstackhelm/heat:2023.1-ubuntu_focal +... diff --git a/values_overrides/mariadb-cluster/2023.2-ubuntu_jammy.yaml b/values_overrides/mariadb-cluster/2023.2-ubuntu_jammy.yaml new file mode 100644 index 0000000000..e234a9e0aa --- /dev/null +++ b/values_overrides/mariadb-cluster/2023.2-ubuntu_jammy.yaml @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + prometheus_mysql_exporter_helm_tests: docker.io/openstackhelm/heat:2023.2-ubuntu_jammy + ks_user: docker.io/openstackhelm/heat:2023.2-ubuntu_jammy +... diff --git a/values_overrides/mariadb-cluster/2024.1-ubuntu_jammy.yaml b/values_overrides/mariadb-cluster/2024.1-ubuntu_jammy.yaml new file mode 100644 index 0000000000..6c87b70789 --- /dev/null +++ b/values_overrides/mariadb-cluster/2024.1-ubuntu_jammy.yaml @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + prometheus_mysql_exporter_helm_tests: docker.io/openstackhelm/heat:2024.1-ubuntu_jammy + ks_user: docker.io/openstackhelm/heat:2024.1-ubuntu_jammy +... diff --git a/values_overrides/mariadb-cluster/2024.2-ubuntu_jammy.yaml b/values_overrides/mariadb-cluster/2024.2-ubuntu_jammy.yaml new file mode 100644 index 0000000000..78d19b0003 --- /dev/null +++ b/values_overrides/mariadb-cluster/2024.2-ubuntu_jammy.yaml @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + prometheus_mysql_exporter_helm_tests: docker.io/openstackhelm/heat:2024.2-ubuntu_jammy + ks_user: docker.io/openstackhelm/heat:2024.2-ubuntu_jammy +... diff --git a/values_overrides/mariadb-cluster/apparmor.yaml b/values_overrides/mariadb-cluster/apparmor.yaml new file mode 100644 index 0000000000..c0fb0d381e --- /dev/null +++ b/values_overrides/mariadb-cluster/apparmor.yaml @@ -0,0 +1,21 @@ +--- +pod: + mandatory_access_control: + type: apparmor + mariadb-server: + init-0: runtime/default + agent: runtime/default + init: runtime/default + metrics: runtime/default + mariadb: runtime/default + mariadb-test: + init: runtime/default + mariadb-test: runtime/default + refresh-statefulset: + init: runtime/default + mariadb-refresh-statefulset: runtime/default + +monitoring: + prometheus: + enabled: true +... diff --git a/values_overrides/mariadb-cluster/downscaled.yaml b/values_overrides/mariadb-cluster/downscaled.yaml new file mode 100644 index 0000000000..e536d1304a --- /dev/null +++ b/values_overrides/mariadb-cluster/downscaled.yaml @@ -0,0 +1,8 @@ +--- +conf: + galera: + enabled: false +pod: + replicas: + server: 1 +... diff --git a/values_overrides/mariadb-cluster/local-storage.yaml b/values_overrides/mariadb-cluster/local-storage.yaml new file mode 100644 index 0000000000..2346728cac --- /dev/null +++ b/values_overrides/mariadb-cluster/local-storage.yaml @@ -0,0 +1,11 @@ +--- +pod: + replicas: + server: 1 +volume: + size: 1Gi + class_name: local-storage +monitoring: + prometheus: + enabled: false +... diff --git a/values_overrides/mariadb-cluster/netpol.yaml b/values_overrides/mariadb-cluster/netpol.yaml new file mode 100644 index 0000000000..7c2ba1f8ed --- /dev/null +++ b/values_overrides/mariadb-cluster/netpol.yaml @@ -0,0 +1,84 @@ +--- +manifests: + network_policy: true +network_policy: + mariadb: + egress: + - to: + - ipBlock: + cidr: %%%REPLACE_API_ADDR%%%/32 + ports: + - protocol: TCP + port: %%%REPLACE_API_PORT%%% + ingress: + - from: + - podSelector: + matchLabels: + application: keystone + - podSelector: + matchLabels: + application: heat + - podSelector: + matchLabels: + application: glance + - podSelector: + matchLabels: + application: cinder + - podSelector: + matchLabels: + application: aodh + - podSelector: + matchLabels: + application: barbican + - podSelector: + matchLabels: + application: ceilometer + - podSelector: + matchLabels: + application: designate + - podSelector: + matchLabels: + application: horizon + - podSelector: + matchLabels: + application: ironic + - podSelector: + matchLabels: + application: magnum + - podSelector: + matchLabels: + application: mistral + - podSelector: + matchLabels: + application: nova + - podSelector: + matchLabels: + application: neutron + - podSelector: + matchLabels: + application: rally + - podSelector: + matchLabels: + application: senlin + - podSelector: + matchLabels: + application: placement + - podSelector: + matchLabels: + application: prometheus-mysql-exporter + - podSelector: + matchLabels: + application: mariadb + - podSelector: + matchLabels: + application: mariadb-backup + ports: + - protocol: TCP + port: 3306 + - protocol: TCP + port: 4567 + - protocol: TCP + port: 80 + - protocol: TCP + port: 8080 +... diff --git a/values_overrides/mariadb-cluster/prometheus.yaml b/values_overrides/mariadb-cluster/prometheus.yaml new file mode 100644 index 0000000000..91093da702 --- /dev/null +++ b/values_overrides/mariadb-cluster/prometheus.yaml @@ -0,0 +1,14 @@ +--- +monitoring: + prometheus: + enabled: true +manifests: + monitoring: + prometheus: + configmap_bin: true + deployment_exporter: true + job_user_create: true + secret_etc: true + service_exporter: true + network_policy_exporter: true +... diff --git a/values_overrides/mariadb-cluster/tls.yaml b/values_overrides/mariadb-cluster/tls.yaml new file mode 100644 index 0000000000..d50f732bfd --- /dev/null +++ b/values_overrides/mariadb-cluster/tls.yaml @@ -0,0 +1,13 @@ +--- +endpoints: + oslo_db: + host_fqdn_override: + default: + tls: + secretName: mariadb-tls-direct + issuerRef: + name: ca-issuer + kind: ClusterIssuer +manifests: + certificates: true +... diff --git a/values_overrides/mariadb-cluster/ubuntu_focal.yaml b/values_overrides/mariadb-cluster/ubuntu_focal.yaml new file mode 100644 index 0000000000..0b69fb00f5 --- /dev/null +++ b/values_overrides/mariadb-cluster/ubuntu_focal.yaml @@ -0,0 +1,20 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + mariadb: docker.io/openstackhelm/mariadb:latest-ubuntu_focal + ks_user: docker.io/openstackhelm/heat:wallaby-ubuntu_focal + scripted_test: docker.io/openstackhelm/mariadb:ubuntu_focal-20210415 + mariadb_cluster_refresh_statefulset: quay.io/airshipit/porthole-mysqlclient-utility:latest-ubuntu_focal +... diff --git a/values_overrides/mariadb-cluster/upscaled.yaml b/values_overrides/mariadb-cluster/upscaled.yaml new file mode 100644 index 0000000000..b35f915508 --- /dev/null +++ b/values_overrides/mariadb-cluster/upscaled.yaml @@ -0,0 +1,8 @@ +--- +conf: + galera: + enabled: true +pod: + replicas: + server: 3 +... diff --git a/values_overrides/mariadb/2023.1-ubuntu_focal.yaml b/values_overrides/mariadb/2023.1-ubuntu_focal.yaml new file mode 100644 index 0000000000..4c9e14eccb --- /dev/null +++ b/values_overrides/mariadb/2023.1-ubuntu_focal.yaml @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + prometheus_mysql_exporter_helm_tests: docker.io/openstackhelm/heat:2023.1-ubuntu_focal + ks_user: docker.io/openstackhelm/heat:2023.1-ubuntu_focal +... diff --git a/values_overrides/mariadb/2024.1-ubuntu_jammy.yaml b/values_overrides/mariadb/2024.1-ubuntu_jammy.yaml new file mode 100644 index 0000000000..6c87b70789 --- /dev/null +++ b/values_overrides/mariadb/2024.1-ubuntu_jammy.yaml @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + prometheus_mysql_exporter_helm_tests: docker.io/openstackhelm/heat:2024.1-ubuntu_jammy + ks_user: docker.io/openstackhelm/heat:2024.1-ubuntu_jammy +... diff --git a/values_overrides/mariadb/2024.2-ubuntu_jammy.yaml b/values_overrides/mariadb/2024.2-ubuntu_jammy.yaml new file mode 100644 index 0000000000..78d19b0003 --- /dev/null +++ b/values_overrides/mariadb/2024.2-ubuntu_jammy.yaml @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + prometheus_mysql_exporter_helm_tests: docker.io/openstackhelm/heat:2024.2-ubuntu_jammy + ks_user: docker.io/openstackhelm/heat:2024.2-ubuntu_jammy +... diff --git a/values_overrides/mariadb/apparmor.yaml b/values_overrides/mariadb/apparmor.yaml new file mode 100644 index 0000000000..09acc7bd63 --- /dev/null +++ b/values_overrides/mariadb/apparmor.yaml @@ -0,0 +1,36 @@ +--- +pod: + mandatory_access_control: + type: apparmor + mariadb-ingress-error-pages: + init: runtime/default + ingress-error-pages: runtime/default + mariadb-ingress: + init: runtime/default + ingress: runtime/default + mariadb-server: + init: runtime/default + mariadb-perms: runtime/default + mariadb: runtime/default + mariadb-backup: + init: runtime/default + mariadb-backup: runtime/default + mariadb-verify-server: runtime/default + mariadb-test: + init: runtime/default + mariadb-test: runtime/default + prometheus-mysql-exporter: + init: runtime/default + mysql-exporter: runtime/default + create-sql-user: + init: runtime/default + exporter-create-sql-user: runtime/default + +monitoring: + prometheus: + enabled: true + +manifests: + cron_job_mariadb_backup: true + job_ks_user: false +... diff --git a/values_overrides/mariadb/backups.yaml b/values_overrides/mariadb/backups.yaml new file mode 100644 index 0000000000..5a7de206c1 --- /dev/null +++ b/values_overrides/mariadb/backups.yaml @@ -0,0 +1,15 @@ +--- +conf: + backup: + enabled: true + remote_backup: + enabled: false +volume: + backup: + enabled: true +manifests: + pvc_backup: true + job_ks_user: false + cron_job_mariadb_backup: true + secret_backup_restore: true +... diff --git a/values_overrides/mariadb/local-storage.yaml b/values_overrides/mariadb/local-storage.yaml new file mode 100644 index 0000000000..2346728cac --- /dev/null +++ b/values_overrides/mariadb/local-storage.yaml @@ -0,0 +1,11 @@ +--- +pod: + replicas: + server: 1 +volume: + size: 1Gi + class_name: local-storage +monitoring: + prometheus: + enabled: false +... diff --git a/values_overrides/mariadb/netpol.yaml b/values_overrides/mariadb/netpol.yaml new file mode 100644 index 0000000000..7c2ba1f8ed --- /dev/null +++ b/values_overrides/mariadb/netpol.yaml @@ -0,0 +1,84 @@ +--- +manifests: + network_policy: true +network_policy: + mariadb: + egress: + - to: + - ipBlock: + cidr: %%%REPLACE_API_ADDR%%%/32 + ports: + - protocol: TCP + port: %%%REPLACE_API_PORT%%% + ingress: + - from: + - podSelector: + matchLabels: + application: keystone + - podSelector: + matchLabels: + application: heat + - podSelector: + matchLabels: + application: glance + - podSelector: + matchLabels: + application: cinder + - podSelector: + matchLabels: + application: aodh + - podSelector: + matchLabels: + application: barbican + - podSelector: + matchLabels: + application: ceilometer + - podSelector: + matchLabels: + application: designate + - podSelector: + matchLabels: + application: horizon + - podSelector: + matchLabels: + application: ironic + - podSelector: + matchLabels: + application: magnum + - podSelector: + matchLabels: + application: mistral + - podSelector: + matchLabels: + application: nova + - podSelector: + matchLabels: + application: neutron + - podSelector: + matchLabels: + application: rally + - podSelector: + matchLabels: + application: senlin + - podSelector: + matchLabels: + application: placement + - podSelector: + matchLabels: + application: prometheus-mysql-exporter + - podSelector: + matchLabels: + application: mariadb + - podSelector: + matchLabels: + application: mariadb-backup + ports: + - protocol: TCP + port: 3306 + - protocol: TCP + port: 4567 + - protocol: TCP + port: 80 + - protocol: TCP + port: 8080 +... diff --git a/values_overrides/mariadb/staggered-backups.yaml b/values_overrides/mariadb/staggered-backups.yaml new file mode 100644 index 0000000000..03412d748c --- /dev/null +++ b/values_overrides/mariadb/staggered-backups.yaml @@ -0,0 +1,38 @@ +--- +conf: + backup: + enabled: true + remote_backup: + enabled: false +pod: + labels: + backup: + staggered_backups: enabled + affinity: + mariadb_backup: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: status.phase + operator: NotIn + values: + - Running + - key: staggered-backups + operator: In + values: + - enabled + namespaces: + - openstack + - osh-infra + - ucp + topologyKey: kubernetes.io/os +volume: + backup: + enabled: true +manifests: + pvc_backup: true + job_ks_user: false + cron_job_mariadb_backup: true + secret_backup_restore: true +... diff --git a/values_overrides/mariadb/tls.yaml b/values_overrides/mariadb/tls.yaml new file mode 100644 index 0000000000..b8da60f899 --- /dev/null +++ b/values_overrides/mariadb/tls.yaml @@ -0,0 +1,23 @@ +--- +pod: + security_context: + server: + container: + perms: + readOnlyRootFilesystem: false + mariadb: + runAsUser: 0 + allowPrivilegeEscalation: true + readOnlyRootFilesystem: false +endpoints: + oslo_db: + host_fqdn_override: + default: + tls: + secretName: mariadb-tls-direct + issuerRef: + name: ca-issuer + kind: ClusterIssuer +manifests: + certificates: true +... diff --git a/values_overrides/mariadb/ubuntu_focal.yaml b/values_overrides/mariadb/ubuntu_focal.yaml new file mode 100644 index 0000000000..cfe1b3da99 --- /dev/null +++ b/values_overrides/mariadb/ubuntu_focal.yaml @@ -0,0 +1,20 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + mariadb: docker.io/openstackhelm/mariadb:latest-ubuntu_focal + prometheus_create_mysql_user: docker.io/library/mariadb:10.5.9-focal + mariadb_backup: quay.io/airshipit/porthole-mysqlclient-utility:latest-ubuntu_focal + scripted_test: docker.io/openstackhelm/mariadb:latest-ubuntu_focal +... diff --git a/values_overrides/mariadb/wait-for-cluster.yaml b/values_overrides/mariadb/wait-for-cluster.yaml new file mode 100644 index 0000000000..f1ecdfce8e --- /dev/null +++ b/values_overrides/mariadb/wait-for-cluster.yaml @@ -0,0 +1,4 @@ +--- +manifests: + job_cluster_wait: true +... diff --git a/values_overrides/memcached/apparmor.yaml b/values_overrides/memcached/apparmor.yaml new file mode 100644 index 0000000000..1d9522289c --- /dev/null +++ b/values_overrides/memcached/apparmor.yaml @@ -0,0 +1,15 @@ +--- +pod: + mandatory_access_control: + type: apparmor + prometheus_memcached_exporter: + init: runtime/default + memcached-exporter: runtime/default + memcached: + init: runtime/default + memcached: runtime/default + +monitoring: + prometheus: + enabled: false +... diff --git a/values_overrides/memcached/netpol.yaml b/values_overrides/memcached/netpol.yaml new file mode 100644 index 0000000000..c4d3079b43 --- /dev/null +++ b/values_overrides/memcached/netpol.yaml @@ -0,0 +1,77 @@ +--- +manifests: + network_policy: true +network_policy: + memcached: + ingress: + - from: + - podSelector: + matchLabels: + application: ingress + - podSelector: + matchLabels: + application: keystone + - podSelector: + matchLabels: + application: heat + - podSelector: + matchLabels: + application: glance + - podSelector: + matchLabels: + application: cinder + - podSelector: + matchLabels: + application: barbican + - podSelector: + matchLabels: + application: ceilometer + - podSelector: + matchLabels: + application: horizon + - podSelector: + matchLabels: + application: ironic + - podSelector: + matchLabels: + application: magnum + - podSelector: + matchLabels: + application: mistral + - podSelector: + matchLabels: + application: nova + - podSelector: + matchLabels: + application: neutron + - podSelector: + matchLabels: + application: senlin + - podSelector: + matchLabels: + application: placement + - podSelector: + matchLabels: + application: prometheus_memcached_exporter + - podSelector: + matchLabels: + application: aodh + - podSelector: + matchLabels: + application: rally + - podSelector: + matchLabels: + application: memcached + ports: + - port: 11211 + protocol: TCP + - port: 9150 + protocol: TCP + egress: + - to: + - ipBlock: + cidr: %%%REPLACE_API_ADDR%%%/32 + ports: + - protocol: TCP + port: %%%REPLACE_API_PORT%%% +... diff --git a/values_overrides/metacontroller/apparmor.yaml b/values_overrides/metacontroller/apparmor.yaml new file mode 100644 index 0000000000..a0670cc21c --- /dev/null +++ b/values_overrides/metacontroller/apparmor.yaml @@ -0,0 +1,7 @@ +--- +pod: + mandatory_access_control: + type: apparmor + metacontroller: + metacontroller: runtime/default +... diff --git a/values_overrides/nagios/apparmor.yaml b/values_overrides/nagios/apparmor.yaml new file mode 100644 index 0000000000..c4aaf760cf --- /dev/null +++ b/values_overrides/nagios/apparmor.yaml @@ -0,0 +1,13 @@ +--- +pod: + mandatory_access_control: + type: apparmor + nagios: + nagios: runtime/default + init: runtime/default + define-nagios-hosts: runtime/default + apache-proxy: runtime/default + nagios-test: + init: runtime/default + nagios-helm-tests: runtime/default +... diff --git a/values_overrides/nagios/elasticsearch-objects.yaml b/values_overrides/nagios/elasticsearch-objects.yaml new file mode 100644 index 0000000000..15e590fea2 --- /dev/null +++ b/values_overrides/nagios/elasticsearch-objects.yaml @@ -0,0 +1,95 @@ +--- +conf: + nagios: + objects: + fluent: + template: | + define service { + check_command check_prom_alert!fluentd_not_running!CRITICAL- fluentd is not running on {instance}!OK- Flunetd is working on all nodes + check_interval 60 + hostgroup_name prometheus-hosts + service_description Fluentd_status + use notifying_service + } + + define service { + check_command check_prom_alert!prom_exporter_fluentd_unavailable!CRITICAL- Fluentd exporter is not collecting metrics for alerting!OK- Fluentd exporter metrics are available. + hostgroup_name prometheus-hosts + service_description Prometheus-exporter_Fluentd + use generic-service + } + elasticsearch: + template: | + define command { + command_line $USER1$/query_elasticsearch.py $USER9$ '$ARG1$' '$ARG2$' '$ARG3$' '$ARG4$' '$ARG5$' --simple_query '$ARG6$' --simple_query_fields '$ARG7$' --match '$ARG8$' --range '$ARG9$' + command_name check_es_query + } + + define command { + command_line $USER1$/query_elasticsearch.py $USER9$ '$ARG1$' '$ARG2$' '$ARG3$' '$ARG4$' '$ARG5$' --simple_query '$ARG6$' --simple_query_fields '$ARG7$' --query_file '/opt/nagios/etc/objects/query_es_clauses.json' --query_clause '$ARG8$' --match '$ARG9$' --range '$ARG10$' + command_name check_es_query_w_file + } + + define service { + check_command check_prom_alert!prom_exporter_elasticsearch_unavailable!CRITICAL- Elasticsearch exporter is not collecting metrics for alerting!OK- Elasticsearch exporter metrics are available. + hostgroup_name prometheus-hosts + service_description Prometheus-exporter_Elasticsearch + use generic-service + } + + define service { + check_command check_prom_alert!es_high_process_open_files_count!CRITICAL- Elasticsearch {host} has high process open file count!OK- Elasticsearch process open file count is normal. + hostgroup_name prometheus-hosts + service_description ES_high-process-open-file-count + use generic-service + } + + define service { + check_command check_prom_alert!es_high_process_cpu_percent!CRITICAL- Elasticsearch {instance} has high process CPU percent!OK- Elasticsearch process cpu usage is normal. + hostgroup_name prometheus-hosts + service_description ES_high-process-cpu-percent + use generic-service + } + + define service { + check_command check_prom_alert!es_fs_usage_high!CRITICAL- Elasticsearch {instance} has high filesystem usage!OK- Elasticsearch filesystem usage is normal. + hostgroup_name prometheus-hosts + service_description ES_high-filesystem-usage + use generic-service + } + + define service { + check_command check_prom_alert!es_unassigned_shards!CRITICAL- Elasticsearch has unassinged shards!OK- Elasticsearch has no unassigned shards. + hostgroup_name prometheus-hosts + service_description ES_unassigned-shards + use generic-service + } + + define service { + check_command check_prom_alert!es_cluster_health_timed_out!CRITICAL- Elasticsearch Cluster health status call timedout!OK- Elasticsearch cluster health is retrievable. + hostgroup_name prometheus-hosts + service_description ES_cluster-health-timedout + use generic-service + } + + define service { + check_command check_prom_alert!es_cluster_health_status_alert!CRITICAL- Elasticsearch cluster health status is not green. One or more shards or replicas are unallocated!OK- Elasticsearch cluster health is green. + hostgroup_name prometheus-hosts + service_description ES_cluster-health-status + use generic-service + } + + define service { + check_command check_prom_alert!es_cluster_health_too_few_nodes_running!CRITICAL- Elasticsearch Cluster has < 3 nodes running!OK- Elasticsearch cluster has 3 or more nodes running. + hostgroup_name prometheus-hosts + service_description ES_cluster-running-node-count + use generic-service + } + + define service { + check_command check_prom_alert!es_cluster_health_too_few_data_nodes_running!CRITICAL- Elasticsearch Cluster has < 3 data nodes running!OK- Elasticsearch cluster has 3 or more data nodes running. + hostgroup_name prometheus-hosts + service_description ES_cluster-running-data-node-count + use generic-service + } +... diff --git a/values_overrides/nagios/openstack-objects.yaml b/values_overrides/nagios/openstack-objects.yaml new file mode 100644 index 0000000000..a6c5d177bd --- /dev/null +++ b/values_overrides/nagios/openstack-objects.yaml @@ -0,0 +1,272 @@ +--- +conf: + nagios: + objects: + mariadb: + template: | + define service { + check_command check_prom_alert!prom_exporter_mariadb_unavailable!CRITICAL- MariaDB exporter is not collecting metrics for alerting!OK- MariaDB exporter metrics are available. + hostgroup_name prometheus-hosts + service_description Prometheus-exporter_MariaDB + use generic-service + } + + define service { + check_command check_prom_alert!mariadb_table_lock_wait_high!CRITICAL- Mariadb has high number of table lock waits!OK- No issues found with table lock waits. + hostgroup_name prometheus-hosts + service_description Mariadb_table-lock-waits-high + use generic-service + } + + define service { + check_command check_prom_alert!mariadb_node_not_ready!CRITICAL- Mariadb {instance} is not ready!OK- All galera cluster nodes are ready. + hostgroup_name prometheus-hosts + service_description Mariadb_node-ready + use generic-service + } + + define service { + check_command check_prom_alert!mariadb_galera_node_out_of_sync!CRITICAL- Mariadb {instance} is out of sync!OK- All galera cluster nodes are in sync + hostgroup_name prometheus-hosts + service_description Mariadb_node-synchronized + use generic-service + } + + define service { + check_command check_prom_alert!mariadb_innodb_replication_fallen_behind!CRITICAL- Innodb replication has fallen behind and not recovering!OK- innodb replication lag is nominal. + hostgroup_name prometheus-hosts + service_description Mariadb_innodb-replication-lag + use generic-service + } + rabbitmq: + template: | + define service { + check_command check_prom_alert!rabbitmq_network_pratitions_detected!CRITICAL- Rabbitmq instance {instance} has network partitions!OK- no network partitions detected in rabbitmq + hostgroup_name prometheus-hosts + service_description Rabbitmq_network-partitions-exist + use generic-service + } + + define service { + check_command check_prom_alert!rabbitmq_down!CRITICAL- Rabbitmq instance {instance} is down!OK- rabbitmq is available + hostgroup_name prometheus-hosts + service_description Rabbitmq_up + use generic-service + } + + define service { + check_command check_prom_alert!rabbitmq_file_descriptor_usage_high!CRITICAL- Rabbitmq instance {instance} has file desciptor usage more than 80 percent!OK- rabbitmq file descriptor usage is normal + hostgroup_name prometheus-hosts + service_description Rabbitmq_file-descriptor-usage + use generic-service + } + + define service { + check_command check_prom_alert!rabbitmq_node_disk_free_alarm!CRITICAL- Rabbitmq instance {instance} has a disk usage alarm!OK- rabbitmq node disk has no alarms + hostgroup_name prometheus-hosts + service_description Rabbitmq_node-disk-alarm + use generic-service + } + + define service { + check_command check_prom_alert!rabbitmq_node_memory_alarm!CRITICAL- Rabbitmq instance {instance} has a memory alarm!OK- rabbitmq node memory has no alarms + hostgroup_name prometheus-hosts + service_description Rabbitmq_node-memory-alarm + use generic-service + } + + define service { + check_command check_prom_alert!rabbitmq_less_than_3_nodes!CRITICAL- Rabbitmq has less than 3 nodes to serve!OK- rabbitmq has atleast 3 nodes serving + hostgroup_name prometheus-hosts + service_description Rabbitmq_high-availability + use generic-service + } + + define service { + check_command check_prom_alert!rabbitmq_queue_messages_returned_high!CRITICAL- Rabbitmq has high percent of messages being returned!OK- rabbitmq messages are consumed and low or no returns exist. + hostgroup_name prometheus-hosts + service_description Rabbitmq_message-return-percent + use generic-service + } + + define service { + check_command check_prom_alert!rabbitmq_consumers_low_utilization!CRITICAL- Rabbitmq consumer message consumption rate is slow!OK- rabbitmq message consumption speed is normal + hostgroup_name prometheus-hosts + service_description Rabbitmq_consumer-utilization + use generic-service + } + + define service { + check_command check_prom_alert!rabbitmq_high_message_load!CRITICAL- Rabbitmq unacknowledged message count is high!OK- rabbitmq unacknowledged message count is high + hostgroup_name prometheus-hosts + service_description Rabbitmq_rabbitmq-queue-health + use generic-service + } + openstack: + template: | + define service { + check_command check_prom_alert!os_glance_api_availability!CRITICAL- Glance API at {url} is not available!OK- Glance API is available + check_interval 60 + hostgroup_name prometheus-hosts + service_description API_glance + use notifying_service + } + + define service { + check_command check_prom_alert!os_nova_api_availability!CRITICAL- Nova API at {url} is not available!OK- Nova API is available + check_interval 60 + hostgroup_name prometheus-hosts + service_description API_nova + use notifying_service + } + + define service { + check_command check_prom_alert!os_keystone_api_availability!CRITICAL- Keystone API at {url} is not available!OK- Keystone API is available + check_interval 60 + hostgroup_name prometheus-hosts + service_description API_keystone + use notifying_service + } + + define service { + check_command check_prom_alert!os_neutron_api_availability!CRITICAL- Neutron API at {url} is not available!OK- Neutron API is available + check_interval 60 + hostgroup_name prometheus-hosts + service_description API_neutron + use notifying_service + } + + define service { + check_command check_prom_alert!os_neutron_metadata_agent_availability!CRITICAL- Some Neutron metadata agents are not available!OK- All the neutron metadata agents are up + check_interval 60 + hostgroup_name prometheus-hosts + service_description Service_neutron-metadata-agent + use notifying_service + } + + define service { + check_command check_prom_alert!os_neutron_openvswitch_agent_availability!CRITICAL- Some Neutron openvswitch agents are not available!OK- All the neutron openvswitch agents are up + check_interval 60 + hostgroup_name prometheus-hosts + service_description Service_neutron-openvswitch-agent + use notifying_service + } + + define service { + check_command check_prom_alert!os_neutron_dhcp_agent_availability!CRITICAL- Some Neutron dhcp agents are not available!OK- All the neutron dhcp agents are up + check_interval 60 + hostgroup_name prometheus-hosts + service_description Service_neutron-dhcp-agent + use notifying_service + } + + define service { + check_command check_prom_alert!os_neutron_l3_agent_availability!CRITICAL- Some Neutron dhcp agents are not available!OK- All the neutron l3 agents are up + check_interval 60 + hostgroup_name prometheus-hosts + service_description Service_neutron-l3-agent + use notifying_service + } + + define service { + check_command check_prom_alert!os_swift_api_availability!CRITICAL- Swift API at {url} is not available!OK- Swift API is available + check_interval 60 + hostgroup_name prometheus-hosts + service_description API_swift + use notifying_service + } + + define service { + check_command check_prom_alert!os_cinder_api_availability!CRITICAL- Cinder API at {url} is not available!OK- Cinder API is available + hostgroup_name prometheus-hosts + service_description API_cinder + use notifying_service + } + + define service { + check_command check_prom_alert!os_heat_api_availability!CRITICAL- Heat API at {url} is not available!OK- Heat API is available + check_interval 60 + hostgroup_name prometheus-hosts + service_description API_heat + use notifying_service + } + + define service { + check_command check_prom_alert!os_cinder_api_availability!CRITICAL- Cinder API at {url} is not available!OK- Cinder API is available + check_interval 60 + hostgroup_name prometheus-hosts + service_description API_cinder + use notifying_service + } + + define service { + check_command check_prom_alert!os_cinder_scheduler_availability!CRITICAL- Cinder scheduler is not available!OK- Cinder scheduler is available + check_interval 60 + hostgroup_name prometheus-hosts + service_description Service_cinder-scheduler + use notifying_service + } + + define service { + check_command check_prom_alert!os_nova_compute_down!CRITICAL- nova-compute services are down on certain hosts!OK- nova-compute services are up on all hosts + check_interval 60 + hostgroup_name prometheus-hosts + service_description Service_nova-compute + use notifying_service + } + + define service { + check_command check_prom_alert!os_nova_conductor_down!CRITICAL- nova-conductor services are down on certain hosts!OK- nova-conductor services are up on all hosts + check_interval 60 + hostgroup_name prometheus-hosts + service_description Service_nova-conductor + use notifying_service + } + + define service { + check_command check_prom_alert!os_nova_consoleauth_down!CRITICAL- nova-consoleauth services are down on certain hosts!OK- nova-consoleauth services are up on all hosts + check_interval 60 + hostgroup_name prometheus-hosts + service_description Service_nova-consoleauth + use notifying_service + } + + define service { + check_command check_prom_alert!openstack_nova_scheduler_down!CRITICAL- nova-scheduler services are down on certain hosts!OK- nova-scheduler services are up on all hosts + check_interval 60 + hostgroup_name prometheus-hosts + service_description Service_nova-scheduler + use notifying_service + } + + define service { + check_command check_prom_alert!os_vm_vcpu_usage_high!CRITICAL- vcpu usage for openstack VMs is more than 80 percent of available!OK- Openstack VMs vcpu usage is less than 80 percent of available. + check_interval 60 + hostgroup_name prometheus-hosts + service_description OS-Total-Quota_VCPU-usage + use notifying_service + } + + define service { + check_command check_prom_alert!os_vm_ram_usage_high!CRITICAL- RAM usage for openstack VMs is more than 80 percent of available!OK- Openstack VMs RAM usage is less than 80 percent of available. + check_interval 60 + hostgroup_name prometheus-hosts + service_description OS-Total-Quota_RAM-usage + use notifying_service + } + + define service { + check_command check_prom_alert!os_vm_disk_usage_high!CRITICAL- Disk usage for openstack VMs is more than 80 percent of available!OK- Openstack VMs Disk usage is less than 80 percent of available. + check_interval 60 + hostgroup_name prometheus-hosts + service_description OS-Total-Quota_Disk-usage + use notifying_service + } + + define service { + check_command check_prom_alert!prom_exporter_openstack_unavailable!CRITICAL- Openstack exporter is not collecting metrics for alerting!OK- Openstack exporter metrics are available. + hostgroup_name prometheus-hosts + service_description Prometheus-exporter_Openstack + use generic-service + } +... diff --git a/values_overrides/nagios/postgresql-objects.yaml b/values_overrides/nagios/postgresql-objects.yaml new file mode 100644 index 0000000000..355b81e1c0 --- /dev/null +++ b/values_overrides/nagios/postgresql-objects.yaml @@ -0,0 +1,34 @@ +--- +conf: + nagios: + objects: + postgresql: + template: | + define service { + check_command check_prom_alert!prom_exporter_postgresql_unavailable!CRITICAL- Postgresql exporter is not collecting metrics for alerting!OK- Postgresql exporter metrics are available. + hostgroup_name prometheus-hosts + service_description Prometheus-exporter_Postgresql + use generic-service + } + + define service { + check_command check_prom_alert!pg_replication_fallen_behind!CRITICAL- Postgres Replication lag is over 2 minutes!OK- postgresql replication lag is nominal. + hostgroup_name prometheus-hosts + service_description Postgresql_replication-lag + use generic-service + } + + define service { + check_command check_prom_alert!pg_connections_too_high!CRITICAL- Postgres has more than 95% of available connections in use.!OK- postgresql open connections are within bounds. + hostgroup_name prometheus-hosts + service_description Postgresql_connections + use generic-service + } + + define service { + check_command check_prom_alert!pg_deadlocks_detected!CRITICAL- Postgres server is experiencing deadlocks!OK- postgresql is not showing any deadlocks. + hostgroup_name prometheus-hosts + service_description Postgresql_deadlocks + use generic-service + } +... diff --git a/values_overrides/nagios/tls.yaml b/values_overrides/nagios/tls.yaml new file mode 100644 index 0000000000..ac964e0c3c --- /dev/null +++ b/values_overrides/nagios/tls.yaml @@ -0,0 +1,17 @@ +--- +endpoints: + monitoring: + scheme: + default: "https" + port: + http: + default: 443 + elasticsearch: + scheme: + default: "https" + port: + http: + default: 443 +manifests: + certificates: true +... diff --git a/values_overrides/openvswitch/apparmor.yaml b/values_overrides/openvswitch/apparmor.yaml new file mode 100644 index 0000000000..5719c83dbe --- /dev/null +++ b/values_overrides/openvswitch/apparmor.yaml @@ -0,0 +1,14 @@ +# NOTE: Enable this with the correct policy +--- +pod: + mandatory_access_control: + type: apparmor + openvswitch-vswitchd: + openvswitch-vswitchd: runtime/default + openvswitch-vswitchd-modules: runtime/default + init: runtime/default + openvswitch-db: + openvswitch-db: runtime/default + openvswitch-db-perms: runtime/default + init: runtime/default +... diff --git a/values_overrides/openvswitch/dpdk-ubuntu_focal.yaml b/values_overrides/openvswitch/dpdk-ubuntu_focal.yaml new file mode 100644 index 0000000000..bc31d2f5a2 --- /dev/null +++ b/values_overrides/openvswitch/dpdk-ubuntu_focal.yaml @@ -0,0 +1,24 @@ +--- +images: + tags: + openvswitch_db_server: docker.io/openstackhelm/openvswitch:latest-ubuntu_focal-dpdk + openvswitch_vswitchd: docker.io/openstackhelm/openvswitch:latest-ubuntu_focal-dpdk +pod: + resources: + enabled: true + ovs: + vswitchd: + requests: + memory: "2Gi" + cpu: "2" + limits: + memory: "2Gi" + cpu: "2" + hugepages-2Mi: "1Gi" +conf: + ovs_dpdk: + enabled: true + hugepages_mountpath: /dev/hugepages + vhostuser_socket_dir: vhostuser + socket_memory: 512 +... diff --git a/values_overrides/openvswitch/dpdk-ubuntu_jammy.yaml b/values_overrides/openvswitch/dpdk-ubuntu_jammy.yaml new file mode 100644 index 0000000000..c489216e03 --- /dev/null +++ b/values_overrides/openvswitch/dpdk-ubuntu_jammy.yaml @@ -0,0 +1,26 @@ +--- +images: + tags: + openvswitch_db_server: docker.io/openstackhelm/openvswitch:latest-ubuntu_jammy-dpdk + openvswitch_vswitchd: docker.io/openstackhelm/openvswitch:latest-ubuntu_jammy-dpdk +pod: + resources: + enabled: true + ovs: + vswitchd: + requests: + memory: "2Gi" + cpu: "2" + limits: + memory: "2Gi" + cpu: "2" + hugepages-2Mi: "1Gi" +conf: + ovs_dpdk: + enabled: true + hugepages_mountpath: /dev/hugepages + vhostuser_socket_dir: vhostuser + socket_memory: 512 + lcore_mask: 0x1 + pmd_cpu_mask: 0x4 +... diff --git a/values_overrides/openvswitch/netpol.yaml b/values_overrides/openvswitch/netpol.yaml new file mode 100644 index 0000000000..7eedf73caf --- /dev/null +++ b/values_overrides/openvswitch/netpol.yaml @@ -0,0 +1,4 @@ +--- +manifests: + network_policy: true +... diff --git a/values_overrides/openvswitch/ovn.yaml b/values_overrides/openvswitch/ovn.yaml new file mode 100644 index 0000000000..964e8227ea --- /dev/null +++ b/values_overrides/openvswitch/ovn.yaml @@ -0,0 +1,5 @@ +--- +conf: + openvswitch_db_server: + ptcp_port: 6640 +... diff --git a/values_overrides/openvswitch/ubuntu_focal.yaml b/values_overrides/openvswitch/ubuntu_focal.yaml new file mode 100644 index 0000000000..0b23e52dd7 --- /dev/null +++ b/values_overrides/openvswitch/ubuntu_focal.yaml @@ -0,0 +1,6 @@ +--- +images: + tags: + openvswitch_db_server: docker.io/openstackhelm/openvswitch:latest-ubuntu_focal + openvswitch_vswitchd: docker.io/openstackhelm/openvswitch:latest-ubuntu_focal +... diff --git a/values_overrides/openvswitch/ubuntu_jammy.yaml b/values_overrides/openvswitch/ubuntu_jammy.yaml new file mode 100644 index 0000000000..eab896ed4c --- /dev/null +++ b/values_overrides/openvswitch/ubuntu_jammy.yaml @@ -0,0 +1,6 @@ +--- +images: + tags: + openvswitch_db_server: docker.io/openstackhelm/openvswitch:latest-ubuntu_jammy + openvswitch_vswitchd: docker.io/openstackhelm/openvswitch:latest-ubuntu_jammy +... diff --git a/values_overrides/openvswitch/vswitchd-probes.yaml b/values_overrides/openvswitch/vswitchd-probes.yaml new file mode 100644 index 0000000000..7df0d69f4f --- /dev/null +++ b/values_overrides/openvswitch/vswitchd-probes.yaml @@ -0,0 +1,11 @@ +--- +pod: + probes: + ovs_vswitch: + ovs_vswitch: + liveness: + exec: + - /bin/bash + - -c + - '/usr/bin/ovs-appctl bond/list; C1=$?; ovs-vsctl --column statistics list interface dpdk_b0s0 | grep -q -E "rx_|tx_"; C2=$?; ovs-vsctl --column statistics list interface dpdk_b0s1 | grep -q -E "rx_|tx_"; C3=$?; exit $(($C1+$C2+$C3))' +... diff --git a/values_overrides/ovn/ubuntu_focal.yaml b/values_overrides/ovn/ubuntu_focal.yaml new file mode 100644 index 0000000000..6c6bf178d4 --- /dev/null +++ b/values_overrides/ovn/ubuntu_focal.yaml @@ -0,0 +1,8 @@ +--- +images: + tags: + ovn_ovsdb_nb: docker.io/openstackhelm/ovn:ubuntu_focal + ovn_ovsdb_sb: docker.io/openstackhelm/ovn:ubuntu_focal + ovn_northd: docker.io/openstackhelm/ovn:ubuntu_focal + ovn_controller: docker.io/openstackhelm/ovn:ubuntu_focal +... diff --git a/values_overrides/ovn/ubuntu_jammy.yaml b/values_overrides/ovn/ubuntu_jammy.yaml new file mode 100644 index 0000000000..8b4269b482 --- /dev/null +++ b/values_overrides/ovn/ubuntu_jammy.yaml @@ -0,0 +1,8 @@ +--- +images: + tags: + ovn_ovsdb_nb: docker.io/openstackhelm/ovn:ubuntu_jammy + ovn_ovsdb_sb: docker.io/openstackhelm/ovn:ubuntu_jammy + ovn_northd: docker.io/openstackhelm/ovn:ubuntu_jammy + ovn_controller: docker.io/openstackhelm/ovn:ubuntu_jammy +... diff --git a/values_overrides/postgresql/2024.1-ubuntu_jammy.yaml b/values_overrides/postgresql/2024.1-ubuntu_jammy.yaml new file mode 100644 index 0000000000..8e1d505beb --- /dev/null +++ b/values_overrides/postgresql/2024.1-ubuntu_jammy.yaml @@ -0,0 +1,17 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + ks_user: docker.io/openstackhelm/heat:2024.1-ubuntu_jammy +... diff --git a/values_overrides/postgresql/2024.2-ubuntu_jammy.yaml b/values_overrides/postgresql/2024.2-ubuntu_jammy.yaml new file mode 100644 index 0000000000..8c16c5dff4 --- /dev/null +++ b/values_overrides/postgresql/2024.2-ubuntu_jammy.yaml @@ -0,0 +1,17 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + ks_user: docker.io/openstackhelm/heat:2024.2-ubuntu_jammy +... diff --git a/values_overrides/postgresql/apparmor.yaml b/values_overrides/postgresql/apparmor.yaml new file mode 100644 index 0000000000..f87f4342c6 --- /dev/null +++ b/values_overrides/postgresql/apparmor.yaml @@ -0,0 +1,21 @@ +--- +pod: + mandatory_access_control: + type: apparmor + postgresql: + postgresql: runtime/default + set-volume-perms: runtime/default + init: runtime/default + prometheus-postgresql-exporter: + postgresql-exporter: runtime/default + init: runtime/default + prometheus-postgresql-exporter-create-user: + prometheus-postgresql-exporter-create-user: runtime/default + init: runtime/default + postgresql-backup: + init: runtime/default + backup-perms: runtime/default + postgresql-backup: runtime/default +manifests: + cron_job_postgresql_backup: true +... diff --git a/values_overrides/postgresql/backups.yaml b/values_overrides/postgresql/backups.yaml new file mode 100644 index 0000000000..499322a810 --- /dev/null +++ b/values_overrides/postgresql/backups.yaml @@ -0,0 +1,15 @@ +--- +conf: + backup: + enabled: true + remote_backup: + enabled: false +volume: + backup: + enabled: true +manifests: + pvc_backup: true + job_ks_user: false + cron_job_postgresql_backup: true + secret_backup_restore: true +... diff --git a/values_overrides/postgresql/netpol.yaml b/values_overrides/postgresql/netpol.yaml new file mode 100644 index 0000000000..3c7edac4a8 --- /dev/null +++ b/values_overrides/postgresql/netpol.yaml @@ -0,0 +1,13 @@ +--- +manifests: + network_policy: true +network_policy: + postgresql: + egress: + - to: + - ipBlock: + cidr: %%%REPLACE_API_ADDR%%%/32 + ports: + - protocol: TCP + port: %%%REPLACE_API_PORT%%% +... diff --git a/values_overrides/postgresql/staggered-backups.yaml b/values_overrides/postgresql/staggered-backups.yaml new file mode 100644 index 0000000000..f51ba78c93 --- /dev/null +++ b/values_overrides/postgresql/staggered-backups.yaml @@ -0,0 +1,38 @@ +--- +conf: + backup: + enabled: true + remote_backup: + enabled: false +pod: + labels: + backup: + staggered_backups: enabled + affinity: + postgresql_backup: + podAntiAffinity: + requiredDuringSchedulingIgnoredDuringExecution: + - labelSelector: + matchExpressions: + - key: status.phase + operator: NotIn + values: + - Running + - key: staggered-backups + operator: In + values: + - enabled + namespaces: + - openstack + - osh-infra + - ucp + topologyKey: kubernetes.io/os +volume: + backup: + enabled: true +manifests: + pvc_backup: true + job_ks_user: false + cron_job_postgresql_backup: true + secret_backup_restore: true +... diff --git a/values_overrides/postgresql/tls.yaml b/values_overrides/postgresql/tls.yaml new file mode 100644 index 0000000000..5ff3a2f51c --- /dev/null +++ b/values_overrides/postgresql/tls.yaml @@ -0,0 +1,26 @@ +--- +conf: + postgresql: + ssl: 'on' +pod: + security_context: + server: + container: + perms: + readOnlyRootFilesystem: false + postgresql: + runAsUser: 0 + allowPrivilegeEscalation: true + readOnlyRootFilesystem: false +endpoints: + postgresql: + host_fqdn_override: + default: + tls: + secretName: postgresql-tls-direct + issuerRef: + name: ca-issuer + kind: ClusterIssuer +manifests: + certificates: true +... diff --git a/values_overrides/powerdns/2023.1-ubuntu_focal.yaml b/values_overrides/powerdns/2023.1-ubuntu_focal.yaml new file mode 100644 index 0000000000..8f56d17867 --- /dev/null +++ b/values_overrides/powerdns/2023.1-ubuntu_focal.yaml @@ -0,0 +1,17 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + db_init: docker.io/openstackhelm/heat:2023.1-ubuntu_focal +... diff --git a/values_overrides/powerdns/2024.1-ubuntu_jammy.yaml b/values_overrides/powerdns/2024.1-ubuntu_jammy.yaml new file mode 100644 index 0000000000..fcb89a48c6 --- /dev/null +++ b/values_overrides/powerdns/2024.1-ubuntu_jammy.yaml @@ -0,0 +1,17 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + db_init: docker.io/openstackhelm/heat:2024.1-ubuntu_jammy +... diff --git a/values_overrides/powerdns/2024.2-ubuntu_jammy.yaml b/values_overrides/powerdns/2024.2-ubuntu_jammy.yaml new file mode 100644 index 0000000000..c4db8cfb34 --- /dev/null +++ b/values_overrides/powerdns/2024.2-ubuntu_jammy.yaml @@ -0,0 +1,17 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + db_init: docker.io/openstackhelm/heat:2024.2-ubuntu_jammy +... diff --git a/values_overrides/prometheus-alertmanager/apparmor.yaml b/values_overrides/prometheus-alertmanager/apparmor.yaml new file mode 100644 index 0000000000..04d3782895 --- /dev/null +++ b/values_overrides/prometheus-alertmanager/apparmor.yaml @@ -0,0 +1,9 @@ +--- +pod: + mandatory_access_control: + type: apparmor + prometheus-alertmanager: + prometheus-alertmanager: runtime/default + prometheus-alertmanager-perms: runtime/default + init: runtime/default +... diff --git a/values_overrides/prometheus-blackbox-exporter/apparmor.yaml b/values_overrides/prometheus-blackbox-exporter/apparmor.yaml new file mode 100644 index 0000000000..12a3ce86a6 --- /dev/null +++ b/values_overrides/prometheus-blackbox-exporter/apparmor.yaml @@ -0,0 +1,7 @@ +--- +pod: + mandatory_access_control: + type: apparmor + prometheus-blackbox-exporter: + blackbox-exporter: runtime/default +... diff --git a/values_overrides/prometheus-kube-state-metrics/apparmor.yaml b/values_overrides/prometheus-kube-state-metrics/apparmor.yaml new file mode 100644 index 0000000000..e77643c633 --- /dev/null +++ b/values_overrides/prometheus-kube-state-metrics/apparmor.yaml @@ -0,0 +1,8 @@ +--- +pod: + mandatory_access_control: + type: apparmor + kube-state-metrics: + kube-state-metrics: runtime/default + init: runtime/default +... diff --git a/values_overrides/prometheus-mysql-exporter/2023.1-ubuntu_focal.yaml b/values_overrides/prometheus-mysql-exporter/2023.1-ubuntu_focal.yaml new file mode 100644 index 0000000000..4c9e14eccb --- /dev/null +++ b/values_overrides/prometheus-mysql-exporter/2023.1-ubuntu_focal.yaml @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + prometheus_mysql_exporter_helm_tests: docker.io/openstackhelm/heat:2023.1-ubuntu_focal + ks_user: docker.io/openstackhelm/heat:2023.1-ubuntu_focal +... diff --git a/values_overrides/prometheus-mysql-exporter/2023.2-ubuntu_jammy.yaml b/values_overrides/prometheus-mysql-exporter/2023.2-ubuntu_jammy.yaml new file mode 100644 index 0000000000..e234a9e0aa --- /dev/null +++ b/values_overrides/prometheus-mysql-exporter/2023.2-ubuntu_jammy.yaml @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + prometheus_mysql_exporter_helm_tests: docker.io/openstackhelm/heat:2023.2-ubuntu_jammy + ks_user: docker.io/openstackhelm/heat:2023.2-ubuntu_jammy +... diff --git a/values_overrides/prometheus-mysql-exporter/2024.1-ubuntu_jammy.yaml b/values_overrides/prometheus-mysql-exporter/2024.1-ubuntu_jammy.yaml new file mode 100644 index 0000000000..6c87b70789 --- /dev/null +++ b/values_overrides/prometheus-mysql-exporter/2024.1-ubuntu_jammy.yaml @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + prometheus_mysql_exporter_helm_tests: docker.io/openstackhelm/heat:2024.1-ubuntu_jammy + ks_user: docker.io/openstackhelm/heat:2024.1-ubuntu_jammy +... diff --git a/values_overrides/prometheus-mysql-exporter/2024.2-ubuntu_jammy.yaml b/values_overrides/prometheus-mysql-exporter/2024.2-ubuntu_jammy.yaml new file mode 100644 index 0000000000..78d19b0003 --- /dev/null +++ b/values_overrides/prometheus-mysql-exporter/2024.2-ubuntu_jammy.yaml @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + prometheus_mysql_exporter_helm_tests: docker.io/openstackhelm/heat:2024.2-ubuntu_jammy + ks_user: docker.io/openstackhelm/heat:2024.2-ubuntu_jammy +... diff --git a/values_overrides/prometheus-mysql-exporter/apparmor.yaml b/values_overrides/prometheus-mysql-exporter/apparmor.yaml new file mode 100644 index 0000000000..fc86fbf8b5 --- /dev/null +++ b/values_overrides/prometheus-mysql-exporter/apparmor.yaml @@ -0,0 +1,37 @@ +--- +pod: + mandatory_access_control: + type: apparmor + mariadb-ingress-error-pages: + init: runtime/default + ingress-error-pages: runtime/default + mariadb-ingress: + init: runtime/default + ingress: runtime/default + mariadb-server: + init-0: runtime/default + agent: runtime/default + init: runtime/default + mariadb-perms: runtime/default + mariadb: runtime/default + mariadb-backup: + init: runtime/default + mariadb-backup: runtime/default + mariadb-verify-server: runtime/default + mariadb-test: + init: runtime/default + mariadb-test: runtime/default + prometheus-mysql-exporter: + init: runtime/default + mysql-exporter: runtime/default + create-sql-user: + init: runtime/default + exporter-create-sql-user: runtime/default + +monitoring: + prometheus: + enabled: true + +manifests: + cron_job_mariadb_backup: true +... diff --git a/values_overrides/prometheus-mysql-exporter/prometheus.yaml b/values_overrides/prometheus-mysql-exporter/prometheus.yaml new file mode 100644 index 0000000000..91093da702 --- /dev/null +++ b/values_overrides/prometheus-mysql-exporter/prometheus.yaml @@ -0,0 +1,14 @@ +--- +monitoring: + prometheus: + enabled: true +manifests: + monitoring: + prometheus: + configmap_bin: true + deployment_exporter: true + job_user_create: true + secret_etc: true + service_exporter: true + network_policy_exporter: true +... diff --git a/values_overrides/prometheus-mysql-exporter/tls.yaml b/values_overrides/prometheus-mysql-exporter/tls.yaml new file mode 100644 index 0000000000..d50f732bfd --- /dev/null +++ b/values_overrides/prometheus-mysql-exporter/tls.yaml @@ -0,0 +1,13 @@ +--- +endpoints: + oslo_db: + host_fqdn_override: + default: + tls: + secretName: mariadb-tls-direct + issuerRef: + name: ca-issuer + kind: ClusterIssuer +manifests: + certificates: true +... diff --git a/values_overrides/prometheus-node-exporter/apparmor.yaml b/values_overrides/prometheus-node-exporter/apparmor.yaml new file mode 100644 index 0000000000..125c15b23f --- /dev/null +++ b/values_overrides/prometheus-node-exporter/apparmor.yaml @@ -0,0 +1,8 @@ +--- +pod: + mandatory_access_control: + type: apparmor + node-exporter: + node-exporter: runtime/default + init: runtime/default +... diff --git a/values_overrides/prometheus-openstack-exporter/apparmor.yaml b/values_overrides/prometheus-openstack-exporter/apparmor.yaml new file mode 100644 index 0000000000..8fd4fadbaf --- /dev/null +++ b/values_overrides/prometheus-openstack-exporter/apparmor.yaml @@ -0,0 +1,11 @@ +--- +pod: + mandatory_access_control: + type: apparmor + prometheus-openstack-exporter: + openstack-metrics-exporter: runtime/default + init: runtime/default + prometheus-openstack-exporter-ks-user: + prometheus-openstack-exporter-ks-user: runtime/default + init: runtime/default +... diff --git a/values_overrides/prometheus-openstack-exporter/netpol.yaml b/values_overrides/prometheus-openstack-exporter/netpol.yaml new file mode 100644 index 0000000000..7eedf73caf --- /dev/null +++ b/values_overrides/prometheus-openstack-exporter/netpol.yaml @@ -0,0 +1,4 @@ +--- +manifests: + network_policy: true +... diff --git a/values_overrides/prometheus-openstack-exporter/tls.yaml b/values_overrides/prometheus-openstack-exporter/tls.yaml new file mode 100644 index 0000000000..99667ca857 --- /dev/null +++ b/values_overrides/prometheus-openstack-exporter/tls.yaml @@ -0,0 +1,4 @@ +--- +manifests: + certificates: true +... diff --git a/values_overrides/prometheus-process-exporter/apparmor.yaml b/values_overrides/prometheus-process-exporter/apparmor.yaml new file mode 100644 index 0000000000..3a955bb62d --- /dev/null +++ b/values_overrides/prometheus-process-exporter/apparmor.yaml @@ -0,0 +1,8 @@ +--- +pod: + mandatory_access_control: + type: apparmor + process-exporter: + process-exporter: runtime/default + init: runtime/default +... diff --git a/values_overrides/prometheus/2023.1-ubuntu_focal.yaml b/values_overrides/prometheus/2023.1-ubuntu_focal.yaml new file mode 100644 index 0000000000..1292734fc6 --- /dev/null +++ b/values_overrides/prometheus/2023.1-ubuntu_focal.yaml @@ -0,0 +1,17 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + helm_tests: docker.io/openstackhelm/heat:2023.1-ubuntu_focal +... diff --git a/values_overrides/prometheus/2024.1-ubuntu_jammy.yaml b/values_overrides/prometheus/2024.1-ubuntu_jammy.yaml new file mode 100644 index 0000000000..efba1791d5 --- /dev/null +++ b/values_overrides/prometheus/2024.1-ubuntu_jammy.yaml @@ -0,0 +1,17 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + helm_tests: docker.io/openstackhelm/heat:2024.1-ubuntu_jammy +... diff --git a/values_overrides/prometheus/2024.2-ubuntu_jammy.yaml b/values_overrides/prometheus/2024.2-ubuntu_jammy.yaml new file mode 100644 index 0000000000..d389163c67 --- /dev/null +++ b/values_overrides/prometheus/2024.2-ubuntu_jammy.yaml @@ -0,0 +1,17 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + helm_tests: docker.io/openstackhelm/heat:2024.2-ubuntu_jammy +... diff --git a/values_overrides/prometheus/alertmanager.yaml b/values_overrides/prometheus/alertmanager.yaml new file mode 100644 index 0000000000..0fc857ced6 --- /dev/null +++ b/values_overrides/prometheus/alertmanager.yaml @@ -0,0 +1,33 @@ +--- +conf: + prometheus: + rules: + alertmanager: + groups: + - name: alertmanager.rules + rules: + - alert: AlertmanagerConfigInconsistent + expr: count_values("config_hash", alertmanager_config_hash) BY (service) / ON(service) GROUP_LEFT() label_replace(prometheus_operator_alertmanager_spec_replicas, "service", "alertmanager-$1", "alertmanager", "(.*)") != 1 + for: 5m + labels: + severity: critical + annotations: + description: The configuration of the instances of the Alertmanager cluster `{{$labels.service}}` are out of sync. + summary: Alertmanager configurations are inconsistent + - alert: AlertmanagerDownOrMissing + expr: label_replace(prometheus_operator_alertmanager_spec_replicas, "job", "alertmanager-$1", "alertmanager", "(.*)") / ON(job) GROUP_RIGHT() sum(up) BY (job) != 1 + for: 5m + labels: + severity: warning + annotations: + description: An unexpected number of Alertmanagers are scraped or Alertmanagers disappeared from discovery. + summary: Alertmanager down or not discovered + - alert: FailedReload + expr: alertmanager_config_last_reload_successful == 0 + for: 10m + labels: + severity: warning + annotations: + description: Reloading Alertmanager's configuration has failed for {{ $labels.namespace }}/{{ $labels.pod }}. + summary: Alertmanager configuration reload has failed +... diff --git a/values_overrides/prometheus/apparmor.yaml b/values_overrides/prometheus/apparmor.yaml new file mode 100644 index 0000000000..bf6f5b6eed --- /dev/null +++ b/values_overrides/prometheus/apparmor.yaml @@ -0,0 +1,13 @@ +--- +pod: + mandatory_access_control: + type: apparmor + prometheus: + prometheus: runtime/default + prometheus-perms: runtime/default + apache-proxy: runtime/default + init: runtime/default + prometheus-test: + prometheus-helm-tests: runtime/default + init: runtime/default +... diff --git a/values_overrides/prometheus/ceph.yaml b/values_overrides/prometheus/ceph.yaml new file mode 100644 index 0000000000..3cadf4b50c --- /dev/null +++ b/values_overrides/prometheus/ceph.yaml @@ -0,0 +1,84 @@ +--- +conf: + prometheus: + rules: + ceph: + groups: + - name: ceph.recording_rules + rules: + - record: ceph_cluster_usage_percent + expr: 100 * (ceph_cluster_total_used_bytes / ceph_cluster_total_bytes) + - record: ceph_placement_group_degrade_percent + expr: 100 * (ceph_pg_degraded / ceph_pg_total) + - record: ceph_osd_down_percent + expr: 100 * (count(ceph_osd_up == 0) / count(ceph_osd_metadata)) + - record: ceph_osd_out_percent + expr: 100 * (count(ceph_osd_in == 0) / count(ceph_osd_metadata)) + - name: ceph.alerting_rules + rules: + - alert: prom_exporter_ceph_unavailable + expr: absent(ceph_health_status) + for: 10m + labels: + severity: warning + annotations: + description: Ceph exporter is not collecting metrics or is not available for past 10 minutes + title: Ceph exporter is not collecting metrics or is not available + - alert: no_active_ceph_mgr + expr: avg_over_time(up{job="ceph-mgr"}[5m]) == 0 + labels: + severity: warning + annotations: + description: 'no ceph active mgr is present or all ceph mgr are down' + summary: 'no ceph active mgt is present' + - alert: ceph_monitor_quorum_low + expr: ceph_mon_quorum_count < 3 + for: 5m + labels: + severity: page + annotations: + description: 'ceph monitor quorum has been less than 3 for more than 5 minutes' + summary: 'ceph high availability is at risk' + - alert: ceph_monitor_quorum_absent + expr: absent(avg_over_time(ceph_mon_quorum_status[5m])) + labels: + severity: page + annotations: + description: 'ceph monitor quorum has been gone for more than 5 minutes' + summary: 'ceph high availability is at risk' + - alert: ceph_cluster_usage_high + expr: avg_over_time(ceph_cluster_usage_percent[5m]) > 80 + labels: + severity: page + annotations: + description: 'ceph cluster capacity usage more than 80 percent' + summary: 'ceph cluster usage is more than 80 percent' + - alert: ceph_placement_group_degrade_pct_high + expr: avg_over_time(ceph_placement_group_degrade_percent[5m]) > 80 + labels: + severity: critical + annotations: + description: 'ceph placement group degradation is more than 80 percent' + summary: 'ceph placement groups degraded' + - alert: ceph_osd_down_pct_high + expr: avg_over_time(ceph_osd_down_percent[5m]) > 80 + labels: + severity: critical + annotations: + description: 'ceph OSDs down percent is more than 80 percent' + summary: 'ceph OSDs down percent is high' + - alert: ceph_osd_down + expr: avg_over_time(ceph_osd_up[5m]) == 0 + labels: + severity: critical + annotations: + description: 'ceph OSD {{ $labels.ceph_daemon }} is down in instance {{ $labels.instance }}.' + summary: 'ceph OSD {{ $labels.ceph_daemon }} is down in instance {{ $labels.instance }}.' + - alert: ceph_osd_out + expr: avg_over_time(ceph_osd_in[5m]) == 0 + labels: + severity: page + annotations: + description: 'ceph OSD {{ $labels.ceph_daemon }} is out in instance {{ $labels.instance }}.' + summary: 'ceph OSD {{ $labels.ceph_daemon }} is out in instance {{ $labels.instance }}.' +... diff --git a/values_overrides/prometheus/elasticsearch.yaml b/values_overrides/prometheus/elasticsearch.yaml new file mode 100644 index 0000000000..965fb163c9 --- /dev/null +++ b/values_overrides/prometheus/elasticsearch.yaml @@ -0,0 +1,101 @@ +--- +conf: + prometheus: + rules: + elasticsearch: + groups: + - name: elasticsearch.alerting_rules + rules: + - alert: prom_exporter_elasticsearch_unavailable + expr: avg_over_time(up{job="elasticsearch-exporter"}[5m]) == 0 + for: 5m + labels: + severity: warning + annotations: + description: Elasticsearch exporter is not collecting metrics or is not available for past 10 minutes + title: Elasticsearch exporter is not collecting metrics or is not available + - alert: es_high_process_open_files_count + expr: sum(elasticsearch_process_open_files_count) by (host) > 64000 + for: 10m + labels: + severity: warning + annotations: + description: 'Elasticsearch at {{ $labels.host }} has more than 64000 process open file count.' + summary: 'Elasticsearch has a very high process open file count.' + - alert: es_high_process_cpu_percent + expr: elasticsearch_process_cpu_percent > 95 + for: 10m + labels: + severity: warning + annotations: + description: 'Elasticsearch at {{ $labels.instance }} has high process cpu percent of {{ $value }}.' + summary: 'Elasticsearch process cpu usage is more than 95 percent.' + - alert: es_fs_usage_high + expr: (100 * (elasticsearch_filesystem_data_size_bytes - elasticsearch_filesystem_data_free_bytes) / elasticsearch_filesystem_data_size_bytes) > 80 + for: 10m + labels: + severity: warning + annotations: + description: 'Elasticsearch at {{ $labels.instance }} has filesystem usage of {{ $value }}.' + summary: 'Elasticsearch filesystem usage is high.' + - alert: es_unassigned_shards + expr: elasticsearch_cluster_health_unassigned_shards > 0 + for: 10m + labels: + severity: warning + annotations: + description: 'Elasticsearch has {{ $value }} unassigned shards.' + summary: 'Elasticsearch has unassigned shards and hence a unhealthy cluster state.' + - alert: es_cluster_health_timed_out + expr: elasticsearch_cluster_health_timed_out > 0 + for: 10m + labels: + severity: warning + annotations: + description: 'Elasticsearch cluster health status call timedout {{ $value }} times.' + summary: 'Elasticsearch cluster health status calls are timing out.' + - alert: es_cluster_health_status_alert + expr: (sum(elasticsearch_cluster_health_status{color="green"})*2)+sum(elasticsearch_cluster_health_status{color="yellow"}) < 2 + for: 10m + labels: + severity: warning + annotations: + description: 'Elasticsearch cluster health status is {{ $value }}, not 2 (green). One or more shards or replicas are unallocated.' + summary: 'Elasticsearch cluster health status is not green.' + - alert: es_cluster_health_too_few_nodes_running + expr: elasticsearch_cluster_health_number_of_nodes < 3 + for: 10m + labels: + severity: warning + annotations: + description: 'There are only {{$value}} < 3 ElasticSearch nodes running' + summary: 'ElasticSearch running on less than 3 nodes' + - alert: es_cluster_health_too_few_data_nodes_running + expr: elasticsearch_cluster_health_number_of_data_nodes < 3 + for: 10m + labels: + severity: warning + annotations: + description: 'There are only {{$value}} < 3 ElasticSearch data nodes running' + summary: 'ElasticSearch running on less than 3 data nodes' + - alert: es_cluster_health_too_few_data_nodes_running + expr: elasticsearch_cluster_health_number_of_data_nodes < 3 + for: 10m + labels: + severity: warning + annotations: + description: 'There are only {{$value}} < 3 ElasticSearch data nodes running' + summary: 'ElasticSearch running on less than 3 data nodes' + fluentd: + groups: + - name: fluentd.alerting_rules + rules: + - alert: prom_exporter_fluentd_unavailable + expr: avg_over_time(up{job="fluentd-daemonset-exporter"}[5m]) == 0 + for: 5m + labels: + severity: warning + annotations: + description: Fluentd exporter is not collecting metrics or is not available for past 10 minutes + title: Fluentd exporter is not collecting metrics or is not available +... diff --git a/values_overrides/prometheus/kubernetes.yaml b/values_overrides/prometheus/kubernetes.yaml new file mode 100644 index 0000000000..8145ef217f --- /dev/null +++ b/values_overrides/prometheus/kubernetes.yaml @@ -0,0 +1,381 @@ +--- +conf: + prometheus: + rules: + kubernetes: + groups: + - name: calico.rules + rules: + - alert: prom_exporter_calico_unavailable + expr: avg_over_time(up{job="kubernetes-pods",application="calico"}[5m]) == 0 + for: 10m + labels: + severity: warning + annotations: + description: Calico exporter is not collecting metrics or is not available for past 10 minutes + title: Calico exporter is not collecting metrics or is not available + - alert: calico_datapane_failures_high_1h + expr: absent(felix_int_dataplane_failures) OR increase(felix_int_dataplane_failures[1h]) > 5 + labels: + severity: page + annotations: + description: 'Felix instance {{ $labels.instance }} has seen {{ $value }} dataplane failures within the last hour' + summary: 'A high number of dataplane failures within Felix are happening' + - alert: calico_datapane_address_msg_batch_size_high_5m + expr: absent(felix_int_dataplane_addr_msg_batch_size_sum) OR absent(felix_int_dataplane_addr_msg_batch_size_count) OR (felix_int_dataplane_addr_msg_batch_size_sum/felix_int_dataplane_addr_msg_batch_size_count) > 5 + for: 5m + labels: + severity: page + annotations: + description: 'Felix instance {{ $labels.instance }} has seen a high value of {{ $value }} dataplane address message batch size' + summary: 'Felix address message batch size is higher' + - alert: calico_datapane_iface_msg_batch_size_high_5m + expr: absent(felix_int_dataplane_iface_msg_batch_size_sum) OR absent(felix_int_dataplane_iface_msg_batch_size_count) OR (felix_int_dataplane_iface_msg_batch_size_sum/felix_int_dataplane_iface_msg_batch_size_count) > 5 + for: 5m + labels: + severity: page + annotations: + description: 'Felix instance {{ $labels.instance }} has seen a high value of {{ $value }} dataplane interface message batch size' + summary: 'Felix interface message batch size is higher' + - alert: calico_ipset_errors_high_1h + expr: absent(felix_ipset_errors) OR increase(felix_ipset_errors[1h]) > 5 + labels: + severity: page + annotations: + description: 'Felix instance {{ $labels.instance }} has seen {{ $value }} ipset errors within the last hour' + summary: 'A high number of ipset errors within Felix are happening' + - alert: calico_iptable_save_errors_high_1h + expr: absent(felix_iptables_save_errors) OR increase(felix_iptables_save_errors[1h]) > 5 + labels: + severity: page + annotations: + description: 'Felix instance {{ $labels.instance }} has seen {{ $value }} iptable save errors within the last hour' + summary: 'A high number of iptable save errors within Felix are happening' + - alert: calico_iptable_restore_errors_high_1h + expr: absent(felix_iptables_restore_errors) OR increase(felix_iptables_restore_errors[1h]) > 5 + labels: + severity: page + annotations: + description: 'Felix instance {{ $labels.instance }} has seen {{ $value }} iptable restore errors within the last hour' + summary: 'A high number of iptable restore errors within Felix are happening' + - name: etcd3.rules + rules: + - alert: etcd_InsufficientMembers + expr: count(up{job="etcd"} == 0) > (count(up{job="etcd"}) / 2 - 1) + for: 3m + labels: + severity: critical + annotations: + description: If one more etcd member goes down the cluster will be unavailable + summary: etcd cluster insufficient members + - alert: etcd_NoLeader + expr: etcd_server_has_leader{job="etcd"} == 0 + for: 1m + labels: + severity: critical + annotations: + description: etcd member {{ $labels.instance }} has no leader + summary: etcd member has no leader + - alert: etcd_HighNumberOfLeaderChanges + expr: increase(etcd_server_leader_changes_seen_total{job="etcd"}[1h]) > 3 + labels: + severity: warning + annotations: + description: etcd instance {{ $labels.instance }} has seen {{ $value }} leader changes within the last hour + summary: a high number of leader changes within the etcd cluster are happening + - alert: etcd_HighNumberOfFailedGRPCRequests + expr: sum(rate(etcd_grpc_requests_failed_total{job="etcd"}[5m])) BY (grpc_method) / sum(rate(etcd_grpc_total{job="etcd"}[5m])) BY (grpc_method) > 0.01 + for: 10m + labels: + severity: warning + annotations: + description: '{{ $value }}% of requests for {{ $labels.grpc_method }} failed on etcd instance {{ $labels.instance }}' + summary: a high number of gRPC requests are failing + - alert: etcd_HighNumberOfFailedGRPCRequests + expr: sum(rate(etcd_grpc_requests_failed_total{job="etcd"}[5m])) BY (grpc_method) / sum(rate(etcd_grpc_total{job="etcd"}[5m])) BY (grpc_method) > 0.05 + for: 5m + labels: + severity: critical + annotations: + description: '{{ $value }}% of requests for {{ $labels.grpc_method }} failed on etcd instance {{ $labels.instance }}' + summary: a high number of gRPC requests are failing + - alert: etcd_GRPCRequestsSlow + expr: histogram_quantile(0.99, rate(etcd_grpc_unary_requests_duration_seconds_bucket[5m])) > 0.15 + for: 10m + labels: + severity: critical + annotations: + description: on etcd instance {{ $labels.instance }} gRPC requests to {{ $labels.grpc_method }} are slow + summary: slow gRPC requests + - alert: etcd_HighNumberOfFailedHTTPRequests + expr: sum(rate(etcd_http_failed_total{job="etcd"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job="etcd"}[5m])) BY (method) > 0.01 + for: 10m + labels: + severity: warning + annotations: + description: '{{ $value }}% of requests for {{ $labels.method }} failed on etcd instance {{ $labels.instance }}' + summary: a high number of HTTP requests are failing + - alert: etcd_HighNumberOfFailedHTTPRequests + expr: sum(rate(etcd_http_failed_total{job="etcd"}[5m])) BY (method) / sum(rate(etcd_http_received_total{job="etcd"}[5m])) BY (method) > 0.05 + for: 5m + labels: + severity: critical + annotations: + description: '{{ $value }}% of requests for {{ $labels.method }} failed on etcd instance {{ $labels.instance }}' + summary: a high number of HTTP requests are failing + - alert: etcd_HTTPRequestsSlow + expr: histogram_quantile(0.99, rate(etcd_http_successful_duration_seconds_bucket[5m])) > 0.15 + for: 10m + labels: + severity: warning + annotations: + description: on etcd instance {{ $labels.instance }} HTTP requests to {{ $labels.method }} are slow + summary: slow HTTP requests + - alert: etcd_EtcdMemberCommunicationSlow + expr: histogram_quantile(0.99, rate(etcd_network_member_round_trip_time_seconds_bucket[5m])) > 0.15 + for: 10m + labels: + severity: warning + annotations: + description: etcd instance {{ $labels.instance }} member communication with {{ $labels.To }} is slow + summary: etcd member communication is slow + - alert: etcd_HighNumberOfFailedProposals + expr: increase(etcd_server_proposals_failed_total{job="etcd"}[1h]) > 5 + labels: + severity: warning + annotations: + description: etcd instance {{ $labels.instance }} has seen {{ $value }} proposal failures within the last hour + summary: a high number of proposals within the etcd cluster are failing + - alert: etcd_HighFsyncDurations + expr: histogram_quantile(0.99, rate(etcd_disk_wal_fsync_duration_seconds_bucket[5m])) > 0.5 + for: 10m + labels: + severity: warning + annotations: + description: etcd instance {{ $labels.instance }} fync durations are high + summary: high fsync durations + - alert: etcd_HighCommitDurations + expr: histogram_quantile(0.99, rate(etcd_disk_backend_commit_duration_seconds_bucket[5m])) > 0.25 + for: 10m + labels: + severity: warning + annotations: + description: etcd instance {{ $labels.instance }} commit durations are high + summary: high commit durations + - name: kubelet.rules + rules: + - alert: K8SNodeNotReady + expr: kube_node_status_condition{condition="Ready", status="unknown"} == 1 or kube_node_status_condition{condition="Ready", status="false"} == 1 + for: 1m + labels: + severity: critical + annotations: + description: The Kubelet on {{ $labels.node }} has not checked in with the API, or has set itself to NotReady, for more than a minute + summary: '{{ $labels.node }} Node status is NotReady and {{ $labels.status }}' + - alert: K8SManyNodesNotReady + expr: count(kube_node_status_condition{condition="Ready", status="unknown"} == 1) > 1 and (count(kube_node_status_condition{condition="Ready", status="unknown"} == 1) / count(kube_node_status_condition{condition="Ready", status="unknown"})) > 0.2 + for: 1m + labels: + severity: critical + annotations: + description: '{{ $value }} Kubernetes nodes (more than 10% are in the NotReady state).' + summary: Many Kubernetes nodes are Not Ready + - alert: K8SManyNodesNotReady + expr: count(kube_node_status_condition{condition="Ready", status="false"} == 1) > 1 and (count(kube_node_status_condition{condition="Ready", status="false"} == 1) / count(kube_node_status_condition{condition="Ready", status="false"})) > 0.2 + for: 1m + labels: + severity: critical + annotations: + description: '{{ $value }} Kubernetes nodes (more than 10% are in the NotReady state).' + summary: Many Kubernetes nodes are Not Ready + - alert: K8SNodesNotReady + expr: count(kube_node_status_condition{condition="Ready", status="false"} == 1) > 0 or count(kube_node_status_condition{condition="Ready", status="unknown"} == 1) > 0 + for: 1m + labels: + severity: critical + annotations: + description: '{{ $value }} nodes are notReady state.' + summary: One or more Kubernetes nodes are Not Ready + - alert: K8SKubeletDown + expr: count(up{job="kubelet"} == 0) / count(up{job="kubelet"}) > 0.03 + for: 1m + labels: + severity: critical + annotations: + description: Prometheus failed to scrape {{ $value }}% of kubelets. + summary: Many Kubelets cannot be scraped + - alert: K8SKubeletDown + expr: absent(up{job="kubelet"} == 1) or count(up{job="kubelet"} == 0) / count(up{job="kubelet"}) > 0.1 + for: 1m + labels: + severity: critical + annotations: + description: Prometheus failed to scrape {{ $value }}% of kubelets, or all Kubelets have disappeared from service discovery. + summary: Many Kubelets cannot be scraped + - alert: K8SKubeletTooManyPods + expr: kubelet_running_pod_count > 100 + labels: + severity: warning + annotations: + description: Kubelet {{$labels.instance}} is running {{$value}} pods, close to the limit of 110 + summary: Kubelet is close to pod limit + - name: kube-apiserver.rules + rules: + - alert: K8SApiserverDown + expr: absent(up{job="apiserver"} == 1) + for: 5m + labels: + severity: critical + annotations: + description: Prometheus failed to scrape API server(s), or all API servers have disappeared from service discovery. + summary: API server unreachable + - alert: K8SApiServerLatency + expr: histogram_quantile(0.99, sum(apiserver_request_latencies_bucket{verb!~"CONNECT|WATCHLIST|WATCH|PROXY"}) WITHOUT (instance, resource)) / 1e+06 > 1 + for: 10m + labels: + severity: warning + annotations: + description: 99th percentile Latency for {{ $labels.verb }} requests to the kube-apiserver is higher than 1s. + summary: Kubernetes apiserver latency is high + - name: kube-controller-manager.rules + rules: + - alert: K8SControllerManagerDown + expr: absent(up{job="kube-controller-manager-discovery"} == 1) + for: 5m + labels: + severity: critical + annotations: + description: There is no running K8S controller manager. Deployments and replication controllers are not making progress. + runbook: https://coreos.com/tectonic/docs/latest/troubleshooting/controller-recovery.html#recovering-a-controller-manager + summary: Controller manager is down + - name: kubernetes-object.rules + rules: + - alert: prom_exporter_kube_state_metrics_unavailable + expr: avg_over_time(up{job="kube-state-metrics"}[5m]) == 0 + for: 5m + labels: + severity: warning + annotations: + description: kube-state-metrics exporter is not collecting metrics or is not available for past 10 minutes + title: kube-state-metrics exporter is not collecting metrics or is not available + - alert: kube_statefulset_replicas_unavailable + expr: kube_statefulset_status_replicas < kube_statefulset_replicas + for: 5m + labels: + severity: page + annotations: + description: 'statefulset {{$labels.statefulset}} has {{$value}} replicas, which is less than desired' + summary: '{{$labels.statefulset}}: has inssuficient replicas.' + - alert: daemonsets_misscheduled + expr: kube_daemonset_status_number_misscheduled > 0 + for: 10m + labels: + severity: warning + annotations: + description: 'Daemonset {{$labels.daemonset}} is running where it is not supposed to run' + summary: 'Daemonsets not scheduled correctly' + - alert: daemonsets_not_scheduled + expr: kube_daemonset_status_desired_number_scheduled - kube_daemonset_status_current_number_scheduled > 0 + for: 10m + labels: + severity: warning + annotations: + description: '{{ $value }} of Daemonset {{$labels.daemonset}} scheduled which is less than desired number' + summary: 'Less than desired number of daemonsets scheduled' + - alert: daemonset_pods_unavailable + expr: kube_daemonset_status_number_unavailable > 0 + for: 10m + labels: + severity: warning + annotations: + description: 'Daemonset {{$labels.daemonset}} currently has pods unavailable' + summary: 'Daemonset pods unavailable, due to one of many reasons' + - alert: deployment_replicas_unavailable + expr: kube_deployment_status_replicas_unavailable > 0 + for: 10m + labels: + severity: page + annotations: + description: 'deployment {{$labels.deployment}} has {{$value}} replicas unavailable' + summary: '{{$labels.deployment}}: has inssuficient replicas.' + - alert: rollingupdate_deployment_replica_less_than_spec_max_unavailable + expr: kube_deployment_status_replicas_available - kube_deployment_spec_strategy_rollingupdate_max_unavailable < 0 + for: 10m + labels: + severity: page + annotations: + description: 'deployment {{$labels.deployment}} has {{$value}} replicas available which is less than specified as max unavailable during a rolling update' + summary: '{{$labels.deployment}}: has inssuficient replicas during a rolling update.' + - alert: job_status_failed + expr: kube_job_status_failed > 0 + for: 10m + labels: + severity: page + annotations: + description: 'Job {{$labels.exported_job}} is in failed status' + summary: '{{$labels.exported_job}} has failed status' + - alert: pod_status_pending + expr: kube_pod_status_phase{phase="Pending"} == 1 + for: 10m + labels: + severity: page + annotations: + description: 'Pod {{$labels.pod}} in namespace {{$labels.namespace}} has been in pending status for more than 10 minutes' + summary: 'Pod {{$labels.pod}} in namespace {{$labels.namespace}} in pending status' + - alert: pod_status_error_image_pull + expr: kube_pod_container_status_waiting_reason {reason="ErrImagePull"} == 1 + for: 10m + labels: + severity: page + annotations: + description: 'Pod {{$labels.pod}} in namespace {{$labels.namespace}} has an Image pull error for more than 10 minutes' + summary: 'Pod {{$labels.pod}} in namespace {{$labels.namespace}} in error status' + - alert: pod_status_error_image_pull_backoff + expr: kube_pod_container_status_waiting_reason {reason="ImagePullBackOff"} == 1 + for: 10m + labels: + severity: page + annotations: + description: 'Pod {{$labels.pod}} in namespace {{$labels.namespace}} has an ImagePullBackOff error for more than 10 minutes' + summary: 'Pod {{$labels.pod}} in namespace {{$labels.namespace}} in error status' + - alert: pod_error_crash_loop_back_off + expr: kube_pod_container_status_waiting_reason {reason="CrashLoopBackOff"} == 1 + for: 10m + labels: + severity: page + annotations: + description: 'Pod {{$labels.pod}} in namespace {{$labels.namespace}} has an CrashLoopBackOff error for more than 10 minutes' + summary: 'Pod {{$labels.pod}} in namespace {{$labels.namespace}} in error status' + - alert: pod_error_config_error + expr: kube_pod_container_status_waiting_reason {reason="CreateContainerConfigError"} == 1 + for: 10m + labels: + severity: page + annotations: + description: 'Pod {{$labels.pod}} in namespace {{$labels.namespace}} has a CreateContainerConfigError error for more than 10 minutes' + summary: 'Pod {{$labels.pod}} in namespace {{$labels.namespace}} in error status' + - alert: replicaset_missing_replicas + expr: kube_replicaset_spec_replicas - kube_replicaset_status_ready_replicas > 0 + for: 10m + labels: + severity: page + annotations: + description: 'Replicaset {{$labels.replicaset}} is missing desired number of replicas for more than 10 minutes' + summary: 'Replicaset {{$labels.replicaset}} is missing replicas' + - alert: pod_container_terminated + expr: kube_pod_container_status_terminated_reason{reason=~"OOMKilled|Error|ContainerCannotRun"} > 0 + for: 10m + labels: + severity: page + annotations: + description: 'Pod {{$labels.pod}} in namespace {{$labels.namespace}} has a container terminated for more than 10 minutes' + summary: 'Pod {{$labels.pod}} in namespace {{$labels.namespace}} in error status' + - alert: volume_claim_capacity_high_utilization + expr: 100 * kubelet_volume_stats_used_bytes / kubelet_volume_stats_capacity_bytes > 80 + for: 5m + labels: + severity: page + annotations: + description: 'volume claim {{$labels.persistentvolumeclaim}} usage has exceeded 80% of total capacity' + summary: '{{$labels.persistentvolumeclaim}} usage has exceeded 80% of total capacity.' +... diff --git a/values_overrides/prometheus/local-storage.yaml b/values_overrides/prometheus/local-storage.yaml new file mode 100644 index 0000000000..4d604da22d --- /dev/null +++ b/values_overrides/prometheus/local-storage.yaml @@ -0,0 +1,9 @@ +--- +pod: + replicas: + prometheus: 1 +storage: + requests: + storage: 1Gi + storage_class: local-storage +... diff --git a/values_overrides/prometheus/nodes.yaml b/values_overrides/prometheus/nodes.yaml new file mode 100644 index 0000000000..41c3e737b6 --- /dev/null +++ b/values_overrides/prometheus/nodes.yaml @@ -0,0 +1,245 @@ +--- +conf: + prometheus: + rules: + nodes: + groups: + - name: node.recording_rules + rules: + - record: node_filesystem_free_percent + expr: 100 * {fstype =~ "xfs|ext[34]"} / node_filesystem_size{fstype =~ "xfs|ext[34]"} + - record: node_ram_usage_percent + expr: 100 * (node_memory_MemFree + node_memory_Buffers + node_memory_Cached) / node_memory_MemTotal + - record: node_swap_usage_percent + expr: 100 * (node_memory_SwapFree + node_memory_SwapCached) / node_memory_SwapTotal + - name: nodes.alerting_rules + rules: + - alert: prom_exporter_node_unavailable + expr: avg_over_time(up{job="node-exporter"}[5m]) == 0 + for: 5m + labels: + severity: warning + annotations: + description: node exporter is not collecting metrics or is not available for past 10 minutes + title: node exporter is not collecting metrics or is not available + - alert: node_filesystem_full_80percent + expr: avg_over_time(node_filesystem_free_percent[2m]) > 80 + for: 5m + labels: + severity: page + annotations: + description: '{{$labels.alias}} device {{$labels.device}} on {{$labels.mountpoint}} + has less than 20% free space left.' + summary: '{{$labels.alias}}: Filesystem is running out of space soon.' + - alert: node_filesystem_full_in_4h + expr: predict_linear(node_filesystem_free{fstype =~ "xfs|ext[34]"}[1h], 4 * 3600) <= 0 + for: 5m + labels: + severity: page + annotations: + description: '{{$labels.alias}} device {{$labels.device}} on {{$labels.mountpoint}} + is running out of space of in approx. 4 hours' + summary: '{{$labels.alias}}: Filesystem is running out of space in 4 hours.' + - alert: node_filedescriptors_full_in_3h + expr: predict_linear(node_filefd_allocated[1h], 3 * 3600) >= node_filefd_maximum + for: 20m + labels: + severity: page + annotations: + description: '{{$labels.alias}} is running out of available file descriptors + in approx. 3 hours' + summary: '{{$labels.alias}} is running out of available file descriptors in + 3 hours.' + - alert: node_load1_90percent + expr: node_load1 / ON(alias) count(node_cpu{mode="system"}) BY (alias) >= 0.9 + for: 1h + labels: + severity: page + annotations: + description: '{{$labels.alias}} is running with > 90% total load for at least + 1h.' + summary: '{{$labels.alias}}: Running on high load.' + - alert: node_cpu_util_90percent + expr: 100 - (avg(irate(node_cpu{mode="idle"}[5m])) BY (alias) * 100) >= 90 + for: 1h + labels: + severity: page + annotations: + description: '{{$labels.alias}} has total CPU utilization over 90% for at least + 1h.' + summary: '{{$labels.alias}}: High CPU utilization.' + - alert: node_ram_using_90percent + expr: avg_over_time(node_ram_usage_percent[2m]) > 90 + for: 30m + labels: + severity: page + annotations: + description: '{{$labels.alias}} is using at least 90% of its RAM for at least + 30 minutes now.' + summary: '{{$labels.alias}}: Using lots of RAM.' + - alert: node_swap_using_80percent + expr: avg_over_time(node_swap_usage_percent[2m]) > 80 + for: 10m + labels: + severity: page + annotations: + description: '{{$labels.alias}} is using 80% of its swap space for at least + 10 minutes now.' + summary: '{{$labels.alias}}: Running out of swap soon.' + - alert: node_high_cpu_load + expr: node_load15 / on(alias) count(node_cpu{mode="system"}) by (alias) >= 0 + for: 1m + labels: + severity: warning + annotations: + description: '{{$labels.alias}} is running with load15 > 1 for at least 5 minutes: {{$value}}' + summary: '{{$labels.alias}}: Running on high load: {{$value}}' + - alert: node_high_memory_load + expr: avg_over_time(node_ram_usage_percent[2m]) > 85 + for: 1m + labels: + severity: warning + annotations: + description: Host memory usage is {{ humanize $value }}%. Reported by + instance {{ $labels.instance }} of job {{ $labels.job }}. + summary: Server memory is almost full + - alert: node_high_storage_load + expr: avg_over_time(node_storage_usage_percent{mountpoint="/"}[2m]) > 85 + for: 30s + labels: + severity: warning + annotations: + description: Host storage usage is {{ humanize $value }}%. Reported by + instance {{ $labels.instance }} of job {{ $labels.job }}. + summary: Server storage is almost full + - alert: node_high_swap + expr: (node_memory_SwapTotal - node_memory_SwapFree) < (node_memory_SwapTotal + * 0.4) + for: 1m + labels: + severity: warning + annotations: + description: Host system has a high swap usage of {{ humanize $value }}. Reported + by instance {{ $labels.instance }} of job {{ $labels.job }}. + summary: Server has a high swap usage + - alert: node_high_network_drop_rcv + expr: node_network_receive_drop{device!="lo"} > 3000 + for: 30s + labels: + severity: warning + annotations: + description: Host system has an unusally high drop in network reception ({{ + humanize $value }}). Reported by instance {{ $labels.instance }} of job {{ + $labels.job }} + summary: Server has a high receive drop + - alert: node_high_network_drop_send + expr: node_network_transmit_drop{device!="lo"} > 3000 + for: 30s + labels: + severity: warning + annotations: + description: Host system has an unusally high drop in network transmission ({{ + humanize $value }}). Reported by instance {{ $labels.instance }} of job {{ + $labels.job }} + summary: Server has a high transmit drop + - alert: node_high_network_errs_rcv + expr: node_network_receive_errs{device!="lo"} > 3000 + for: 30s + labels: + severity: warning + annotations: + description: Host system has an unusally high error rate in network reception + ({{ humanize $value }}). Reported by instance {{ $labels.instance }} of job + {{ $labels.job }} + summary: Server has unusual high reception errors + - alert: node_high_network_errs_send + expr: node_network_transmit_errs{device!="lo"} > 3000 + for: 30s + labels: + severity: warning + annotations: + description: Host system has an unusally high error rate in network transmission + ({{ humanize $value }}). Reported by instance {{ $labels.instance }} of job + {{ $labels.job }} + summary: Server has unusual high transmission errors + - alert: node_network_conntrack_usage_80percent + expr: sort(node_nf_conntrack_entries{job="node-exporter"} > node_nf_conntrack_entries_limit{job="node-exporter"} * 0.8) + for: 5m + labels: + severity: page + annotations: + description: '{{$labels.instance}} has network conntrack entries of {{ $value }} which is more than 80% of maximum limit' + summary: '{{$labels.instance}}: available network conntrack entries are low.' + - alert: node_entropy_available_low + expr: node_entropy_available_bits < 300 + for: 5m + labels: + severity: page + annotations: + description: '{{$labels.instance}} has available entropy bits of {{ $value }} which is less than required of 300' + summary: '{{$labels.instance}}: is low on entropy bits.' + - alert: node_hwmon_high_cpu_temp + expr: node_hwmon_temp_crit_celsius*0.9 - node_hwmon_temp_celsius < 0 OR node_hwmon_temp_max_celsius*0.95 - node_hwmon_temp_celsius < 0 + for: 5m + labels: + severity: page + annotations: + description: '{{$labels.alias}} reports hwmon sensor {{$labels.sensor}}/{{$labels.chip}} temperature value is nearly critical: {{$value}}' + summary: '{{$labels.alias}}: Sensor {{$labels.sensor}}/{{$labels.chip}} temp is high: {{$value}}' + - alert: node_vmstat_paging_rate_high + expr: irate(node_vmstat_pgpgin[5m]) > 80 + for: 5m + labels: + severity: page + annotations: + description: '{{$labels.alias}} has a memory paging rate of change higher than 80%: {{$value}}' + summary: '{{$labels.alias}}: memory paging rate is high: {{$value}}' + - alert: node_xfs_block_allocation_high + expr: 100*(node_xfs_extent_allocation_blocks_allocated_total{job="node-exporter", instance=~"172.17.0.1.*"} / (node_xfs_extent_allocation_blocks_freed_total{job="node-exporter", instance=~"172.17.0.1.*"} + node_xfs_extent_allocation_blocks_allocated_total{job="node-exporter", instance=~"172.17.0.1.*"})) > 80 + for: 5m + labels: + severity: page + annotations: + description: '{{$labels.alias}} has xfs allocation blocks higher than 80%: {{$value}}' + summary: '{{$labels.alias}}: xfs block allocation high: {{$value}}' + - alert: node_network_bond_slaves_down + expr: node_net_bonding_slaves - node_net_bonding_slaves_active > 0 + for: 5m + labels: + severity: page + annotations: + description: '{{ $labels.master }} is missing {{ $value }} slave interface(s).' + summary: 'Instance {{ $labels.instance }}: {{ $labels.master }} missing {{ $value }} slave interface(s)' + - alert: node_numa_memory_used + expr: 100*node_memory_numa_MemUsed / node_memory_numa_MemTotal > 80 + for: 5m + labels: + severity: page + annotations: + description: '{{$labels.alias}} has more than 80% NUMA memory usage: {{ $value }}' + summary: '{{$labels.alias}}: has high NUMA memory usage: {{$value}}' + - alert: node_ntp_clock_skew_high + expr: abs(node_ntp_drift_seconds) > 2 + for: 5m + labels: + severity: page + annotations: + description: '{{$labels.alias}} has time difference of more than 2 seconds compared to NTP server: {{ $value }}' + summary: '{{$labels.alias}}: time is skewed by : {{$value}} seconds' + - alert: node_disk_read_latency + expr: (rate(node_disk_read_time_ms[5m]) / rate(node_disk_reads_completed[5m])) > 40 + for: 5m + labels: + severity: page + annotations: + description: '{{$labels.device}} has a high read latency of {{ $value }}' + summary: 'High read latency observed for device {{ $labels.device }}' + - alert: node_disk_write_latency + expr: (rate(node_disk_write_time_ms[5m]) / rate(node_disk_writes_completed[5m])) > 40 + for: 5m + labels: + severity: page + annotations: + description: '{{$labels.device}} has a high write latency of {{ $value }}' + summary: 'High write latency observed for device {{ $labels.device }}' +... diff --git a/values_overrides/prometheus/openstack.yaml b/values_overrides/prometheus/openstack.yaml new file mode 100644 index 0000000000..e7c3db80ea --- /dev/null +++ b/values_overrides/prometheus/openstack.yaml @@ -0,0 +1,325 @@ +--- +conf: + prometheus: + rules: + openstack: + groups: + - name: mariadb.rules + rules: + - alert: prom_exporter_mariadb_openstack_unavailable + expr: avg_over_time(up{job="mysql-exporter",kubernetes_namespace="openstack"}[5m]) == 0 + for: 5m + labels: + severity: warning + annotations: + description: MariaDB exporter in {{ $labels.kubernetes_namespace }} is not collecting metrics or is not available for past 10 minutes + title: MariaDB exporter is not collecting metrics or is not available + - alert: prom_exporter_mariadb_osh_infra_unavailable + expr: avg_over_time(up{job="mysql-exporter",kubernetes_namespace="osh-infra"}[5m]) == 0 + for: 5m + labels: + severity: warning + annotations: + description: MariaDB exporter in {{ $labels.kubernetes_namespace }} is not collecting metrics or is not available for past 10 minutes + title: MariaDB exporter is not collecting metrics or is not available + - alert: mariadb_table_lock_wait_high + expr: 100 * mysql_global_status_table_locks_waited/(mysql_global_status_table_locks_waited + mysql_global_status_table_locks_immediate) > 30 + for: 10m + labels: + severity: warning + annotations: + description: 'Mariadb has high table lock waits of {{ $value }} percentage' + summary: 'Mariadb table lock waits are high' + - alert: mariadb_node_not_ready + expr: mysql_global_status_wsrep_ready != 1 + for: 10m + labels: + severity: warning + annotations: + description: '{{$labels.job}} on {{$labels.instance}} is not ready.' + summary: 'Galera cluster node not ready' + - alert: mariadb_galera_node_out_of_sync + expr: mysql_global_status_wsrep_local_state != 4 AND mysql_global_variables_wsrep_desync == 0 + for: 10m + labels: + severity: warning + annotations: + description: '{{$labels.job}} on {{$labels.instance}} is not in sync ({{$value}} != 4)' + summary: 'Galera cluster node out of sync' + - alert: mariadb_innodb_replication_fallen_behind + expr: (mysql_global_variables_innodb_replication_delay > 30) AND on (instance) (predict_linear(mysql_global_variables_innodb_replication_delay[5m], 60*2) > 0) + for: 10m + labels: + severity: warning + annotations: + description: 'The mysql innodb replication has fallen behind and is not recovering' + summary: 'MySQL innodb replication is lagging' + - name: openstack.rules + rules: + - alert: prom_exporter_openstack_unavailable + expr: avg_over_time(up{job="openstack-metrics"}[5m]) == 0 + for: 5m + labels: + severity: warning + annotations: + description: Openstack exporter is not collecting metrics or is not available for past 10 minutes + title: Openstack exporter is not collecting metrics or is not available + - alert: os_glance_api_availability + expr: openstack_check_glance_api != 1 + for: 5m + labels: + severity: page + annotations: + description: 'Glance API is not available at {{$labels.url}} for more than 5 minutes' + summary: 'Glance API is not available at {{$labels.url}}' + - alert: os_nova_api_availability + expr: openstack_check_nova_api != 1 + for: 5m + labels: + severity: page + annotations: + description: 'Nova API is not available at {{$labels.url}} for more than 5 minutes' + summary: 'Nova API is not available at {{$labels.url}}' + - alert: os_keystone_api_availability + expr: openstack_check_keystone_api != 1 + for: 5m + labels: + severity: page + annotations: + description: 'Keystone API is not available at {{$labels.url}} for more than 5 minutes' + summary: 'Keystone API is not available at {{$labels.url}}' + - alert: os_neutron_api_availability + expr: openstack_check_neutron_api != 1 + for: 5m + labels: + severity: page + annotations: + description: 'Neutron API is not available at {{$labels.url}} for more than 5 minutes' + summary: 'Neutron API is not available at {{$labels.url}}' + - alert: os_neutron_metadata_agent_availability + expr: openstack_services_neutron_metadata_agent_down_total > 0 + for: 5m + labels: + severity: page + annotations: + description: 'One or more neutron metadata_agents are not available for more than 5 minutes' + summary: 'One or more neutron metadata_agents are not available' + - alert: os_neutron_openvswitch_agent_availability + expr: openstack_services_neutron_openvswitch_agent_down_total > 0 + for: 5m + labels: + severity: page + annotations: + description: 'One or more neutron openvswitch agents are not available for more than 5 minutes' + summary: 'One or more neutron openvswitch agents are not available' + - alert: os_neutron_dhcp_agent_availability + expr: openstack_services_neutron_dhcp_agent_down_total > 0 + for: 5m + labels: + severity: page + annotations: + description: 'One or more neutron dhcp agents are not available for more than 5 minutes' + summary: 'One or more neutron dhcp agents are not available' + - alert: os_neutron_l3_agent_availability + expr: openstack_services_neutron_l3_agent_down_total > 0 + for: 5m + labels: + severity: page + annotations: + description: 'One or more neutron L3 agents are not available for more than 5 minutes' + summary: 'One or more neutron L3 agents are not available' + - alert: os_swift_api_availability + expr: openstack_check_swift_api != 1 + for: 5m + labels: + severity: page + annotations: + description: 'Swift API is not available at {{$labels.url}} for more than 5 minutes' + summary: 'Swift API is not available at {{$labels.url}}' + - alert: os_cinder_api_availability + expr: openstack_check_cinder_api != 1 + for: 5m + labels: + severity: page + annotations: + description: 'Cinder API is not available at {{$labels.url}} for more than 5 minutes' + summary: 'Cinder API is not available at {{$labels.url}}' + - alert: os_cinder_scheduler_availability + expr: openstack_services_cinder_cinder_scheduler != 1 + for: 5m + labels: + severity: page + annotations: + description: 'Cinder scheduler is not available for more than 5 minutes' + summary: 'Cinder scheduler is not available' + - alert: os_heat_api_availability + expr: openstack_check_heat_api != 1 + for: 5m + labels: + severity: page + annotations: + description: 'Heat API is not available at {{$labels.url}} for more than 5 minutes' + summary: 'Heat API is not available at {{$labels.url}}' + - alert: os_nova_compute_disabled + expr: openstack_services_nova_compute_disabled_total > 0 + for: 5m + labels: + severity: page + annotations: + description: 'nova-compute is disabled on certain hosts for more than 5 minutes' + summary: 'Openstack compute service nova-compute is disabled on some hosts' + - alert: os_nova_conductor_disabled + expr: openstack_services_nova_conductor_disabled_total > 0 + for: 5m + labels: + severity: page + annotations: + description: 'nova-conductor is disabled on certain hosts for more than 5 minutes' + summary: 'Openstack compute service nova-conductor is disabled on some hosts' + - alert: os_nova_consoleauth_disabled + expr: openstack_services_nova_consoleauth_disabled_total > 0 + for: 5m + labels: + severity: page + annotations: + description: 'nova-consoleauth is disabled on certain hosts for more than 5 minutes' + summary: 'Openstack compute service nova-consoleauth is disabled on some hosts' + - alert: os_nova_scheduler_disabled + expr: openstack_services_nova_scheduler_disabled_total > 0 + for: 5m + labels: + severity: page + annotations: + description: 'nova-scheduler is disabled on certain hosts for more than 5 minutes' + summary: 'Openstack compute service nova-scheduler is disabled on some hosts' + - alert: os_nova_compute_down + expr: openstack_services_nova_compute_down_total > 0 + for: 5m + labels: + severity: page + annotations: + description: 'nova-compute is down on certain hosts for more than 5 minutes' + summary: 'Openstack compute service nova-compute is down on some hosts' + - alert: os_nova_conductor_down + expr: openstack_services_nova_conductor_down_total > 0 + for: 5m + labels: + severity: page + annotations: + description: 'nova-conductor is down on certain hosts for more than 5 minutes' + summary: 'Openstack compute service nova-conductor is down on some hosts' + - alert: os_nova_consoleauth_down + expr: openstack_services_nova_consoleauth_down_total > 0 + for: 5m + labels: + severity: page + annotations: + description: 'nova-consoleauth is down on certain hosts for more than 5 minutes' + summary: 'Openstack compute service nova-consoleauth is down on some hosts' + - alert: os_nova_scheduler_down + expr: openstack_services_nova_scheduler_down_total > 0 + for: 5m + labels: + severity: page + annotations: + description: 'nova-scheduler is down on certain hosts for more than 5 minutes' + summary: 'Openstack compute service nova-scheduler is down on some hosts' + - alert: os_vm_vcpu_usage_high + expr: openstack_total_used_vcpus * 100/(openstack_total_used_vcpus + openstack_total_free_vcpus) > 80 + for: 5m + labels: + severity: page + annotations: + description: 'Openstack VM vcpu usage is hight at {{$value}} percent' + summary: 'Openstack VM vcpu usage is high' + - alert: os_vm_ram_usage_high + expr: openstack_total_used_ram_MB * 100/(openstack_total_used_ram_MB + openstack_total_free_ram_MB) > 80 + for: 5m + labels: + severity: page + annotations: + description: 'Openstack VM RAM usage is hight at {{$value}} percent' + summary: 'Openstack VM RAM usage is high' + - alert: os_vm_disk_usage_high + expr: openstack_total_used_disk_GB * 100/ ( openstack_total_used_disk_GB + openstack_total_free_disk_GB ) > 80 + for: 5m + labels: + severity: page + annotations: + description: 'Openstack VM Disk usage is hight at {{$value}} percent' + summary: 'Openstack VM Disk usage is high' + - name: rabbitmq.rules + rules: + - alert: rabbitmq_network_pratitions_detected + expr: min(partitions) by(instance) > 0 + for: 10m + labels: + severity: warning + annotations: + description: 'RabbitMQ at {{ $labels.instance }} has {{ $value }} partitions' + summary: 'RabbitMQ Network partitions detected' + - alert: rabbitmq_down + expr: min(rabbitmq_up) by(instance) != 1 + for: 10m + labels: + severity: page + annotations: + description: 'RabbitMQ Server instance {{ $labels.instance }} is down' + summary: 'The RabbitMQ Server instance at {{ $labels.instance }} has been down the last 10 mins' + - alert: rabbitmq_file_descriptor_usage_high + expr: fd_used * 100 /fd_total > 80 + for: 10m + labels: + severity: warning + annotations: + description: 'RabbitMQ Server instance {{ $labels.instance }} has high file descriptor usage of {{ $value }} percent.' + summary: 'RabbitMQ file descriptors usage is high for last 10 mins' + - alert: rabbitmq_node_disk_free_alarm + expr: node_disk_free_alarm > 0 + for: 10m + labels: + severity: warning + annotations: + description: 'RabbitMQ Server instance {{ $labels.instance }} has low disk free space available.' + summary: 'RabbitMQ disk space usage is high' + - alert: rabbitmq_node_memory_alarm + expr: node_mem_alarm > 0 + for: 10m + labels: + severity: warning + annotations: + description: 'RabbitMQ Server instance {{ $labels.instance }} has low free memory.' + summary: 'RabbitMQ memory usage is high' + - alert: rabbitmq_less_than_3_nodes + expr: running < 3 + for: 10m + labels: + severity: warning + annotations: + description: 'RabbitMQ Server has less than 3 nodes running.' + summary: 'RabbitMQ server is at risk of loosing data' + - alert: rabbitmq_queue_messages_returned_high + expr: queue_messages_returned_total/queue_messages_published_total * 100 > 50 + for: 5m + labels: + severity: warning + annotations: + description: 'RabbitMQ Server is returing more than 50 percent of messages received.' + summary: 'RabbitMQ server is returning more than 50 percent of messages received.' + - alert: rabbitmq_consumers_low_utilization + expr: queue_consumer_utilisation < .4 + for: 5m + labels: + severity: warning + annotations: + description: 'RabbitMQ consumers message consumption speed is low' + summary: 'RabbitMQ consumers message consumption speed is low' + - alert: rabbitmq_high_message_load + expr: queue_messages_total > 17000 or increase(queue_messages_total[5m]) > 4000 + for: 5m + labels: + severity: warning + annotations: + description: 'RabbitMQ has high message load. Total Queue depth > 17000 or growth more than 4000 messages.' + summary: 'RabbitMQ has high message load' +... diff --git a/values_overrides/prometheus/postgresql.yaml b/values_overrides/prometheus/postgresql.yaml new file mode 100644 index 0000000000..1d68981ca8 --- /dev/null +++ b/values_overrides/prometheus/postgresql.yaml @@ -0,0 +1,41 @@ +--- +conf: + prometheus: + rules: + postgresql: + groups: + - name: postgresql.rules + rules: + - alert: prom_exporter_postgresql_unavailable + expr: avg_over_time(up{job="postgresql-exporter"}[5m]) == 0 + for: 5m + labels: + severity: warning + annotations: + description: postgresql exporter is not collecting metrics or is not available for past 10 minutes + title: postgresql exporter is not collecting metrics or is not available + - alert: pg_replication_fallen_behind + expr: (pg_replication_lag > 120) and ON(instance) (pg_replication_is_replica == 1) + for: 5m + labels: + severity: warning + annotations: + description: Replication lag on server {{$labels.instance}} is currently {{$value | humanizeDuration }} + title: Postgres Replication lag is over 2 minutes + - alert: pg_connections_too_high + expr: sum(pg_stat_activity_count) BY (environment, fqdn) > ON(fqdn) pg_settings_max_connections * 0.95 + for: 5m + labels: + severity: warn + channel: database + annotations: + title: Postgresql has {{$value}} connections on {{$labels.fqdn}} which is close to the maximum + - alert: pg_deadlocks_detected + expr: sum by(datname) (rate(pg_stat_database_deadlocks[1m])) > 0 + for: 5m + labels: + severity: warn + annotations: + description: postgresql at {{$labels.instance}} is showing {{$value}} rate of deadlocks for database {{$labels.datname}} + title: Postgres server is experiencing deadlocks +... diff --git a/values_overrides/prometheus/tls.yaml b/values_overrides/prometheus/tls.yaml new file mode 100644 index 0000000000..7f65b4c2d2 --- /dev/null +++ b/values_overrides/prometheus/tls.yaml @@ -0,0 +1,250 @@ +--- +endpoints: + monitoring: + host_fqdn_override: + default: + tls: + secretName: prometheus-tls-api + issuerRef: + name: ca-issuer + kind: ClusterIssuer + scheme: + default: "https" + port: + http: + default: 443 +network: + prometheus: + ingress: + annotations: + nginx.ingress.kubernetes.io/backend-protocol: https +conf: + httpd: | + ServerRoot "/usr/local/apache2" + Listen 443 + LoadModule mpm_event_module modules/mod_mpm_event.so + LoadModule authn_file_module modules/mod_authn_file.so + LoadModule authn_core_module modules/mod_authn_core.so + LoadModule authz_host_module modules/mod_authz_host.so + LoadModule authz_groupfile_module modules/mod_authz_groupfile.so + LoadModule authz_user_module modules/mod_authz_user.so + LoadModule authz_core_module modules/mod_authz_core.so + LoadModule access_compat_module modules/mod_access_compat.so + LoadModule auth_basic_module modules/mod_auth_basic.so + LoadModule ldap_module modules/mod_ldap.so + LoadModule authnz_ldap_module modules/mod_authnz_ldap.so + LoadModule reqtimeout_module modules/mod_reqtimeout.so + LoadModule filter_module modules/mod_filter.so + LoadModule proxy_html_module modules/mod_proxy_html.so + LoadModule log_config_module modules/mod_log_config.so + LoadModule env_module modules/mod_env.so + LoadModule headers_module modules/mod_headers.so + LoadModule setenvif_module modules/mod_setenvif.so + LoadModule version_module modules/mod_version.so + LoadModule proxy_module modules/mod_proxy.so + LoadModule proxy_connect_module modules/mod_proxy_connect.so + LoadModule proxy_http_module modules/mod_proxy_http.so + LoadModule proxy_balancer_module modules/mod_proxy_balancer.so + LoadModule slotmem_shm_module modules/mod_slotmem_shm.so + LoadModule slotmem_plain_module modules/mod_slotmem_plain.so + LoadModule unixd_module modules/mod_unixd.so + LoadModule status_module modules/mod_status.so + LoadModule autoindex_module modules/mod_autoindex.so + LoadModule ssl_module modules/mod_ssl.so + + + User daemon + Group daemon + + + + AllowOverride none + Require all denied + + + + Require all denied + + + ErrorLog /dev/stderr + + LogLevel warn + + + LogFormat "%a %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" combined + LogFormat "%{X-Forwarded-For}i %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\"" proxy + LogFormat "%h %l %u %t \"%r\" %>s %b" common + + + LogFormat "%a %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %I %O" combinedio + + + SetEnvIf X-Forwarded-For "^.*\..*\..*\..*" forwarded + CustomLog /dev/stdout common + CustomLog /dev/stdout combined + CustomLog /dev/stdout proxy env=forwarded + + + + AllowOverride None + Options None + Require all granted + + + + RequestHeader unset Proxy early + + + + Include conf/extra/proxy-html.conf + + + + # Expose metrics to all users, as this is not sensitive information and + # circumvents the inability of Prometheus to interpolate environment vars + # in its configuration file + + ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/metrics + ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/metrics + Satisfy Any + Allow from all + + # Expose the /federate endpoint to all users, as this is also not + # sensitive information and circumvents the inability of Prometheus to + # interpolate environment vars in its configuration file + + ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/metrics + ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/metrics + Satisfy Any + Allow from all + + # Restrict general user (LDAP) access to the /graph endpoint, as general trusted + # users should only be able to query Prometheus for metrics and not have access + # to information like targets, configuration, flags or build info for Prometheus + + ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/ + ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/ + AuthName "Prometheus" + AuthType Basic + AuthBasicProvider file ldap + AuthUserFile /usr/local/apache2/conf/.htpasswd + AuthLDAPBindDN {{ .Values.endpoints.ldap.auth.admin.bind }} + AuthLDAPBindPassword {{ .Values.endpoints.ldap.auth.admin.password }} + AuthLDAPURL {{ tuple "ldap" "default" "ldap" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | quote }} + Require valid-user + + + ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/graph + ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/graph + AuthName "Prometheus" + AuthType Basic + AuthBasicProvider file ldap + AuthUserFile /usr/local/apache2/conf/.htpasswd + AuthLDAPBindDN {{ .Values.endpoints.ldap.auth.admin.bind }} + AuthLDAPBindPassword {{ .Values.endpoints.ldap.auth.admin.password }} + AuthLDAPURL {{ tuple "ldap" "default" "ldap" . | include "helm-toolkit.endpoints.keystone_endpoint_uri_lookup" | quote }} + Require valid-user + + # Restrict access to the /config (dashboard) and /api/v1/status/config (http) endpoints + # to the admin user + + ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/config + ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/config + AuthName "Prometheus" + AuthType Basic + AuthBasicProvider file + AuthUserFile /usr/local/apache2/conf/.htpasswd + Require valid-user + + + ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/api/v1/status/config + ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/api/v1/status/config + AuthName "Prometheus" + AuthType Basic + AuthBasicProvider file + AuthUserFile /usr/local/apache2/conf/.htpasswd + Require valid-user + + # Restrict access to the /flags (dashboard) and /api/v1/status/flags (http) endpoints + # to the admin user + + ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/flags + ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/flags + AuthName "Prometheus" + AuthType Basic + AuthBasicProvider file + AuthUserFile /usr/local/apache2/conf/.htpasswd + Require valid-user + + + ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/api/v1/status/flags + ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/api/v1/status/flags + AuthName "Prometheus" + AuthType Basic + AuthBasicProvider file + AuthUserFile /usr/local/apache2/conf/.htpasswd + Require valid-user + + # Restrict access to the /status (dashboard) endpoint to the admin user + + ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/status + ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/status + AuthName "Prometheus" + AuthType Basic + AuthBasicProvider file + AuthUserFile /usr/local/apache2/conf/.htpasswd + Require valid-user + + # Restrict access to the /rules (dashboard) endpoint to the admin user + + ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/rules + ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/rules + AuthName "Prometheus" + AuthType Basic + AuthBasicProvider file + AuthUserFile /usr/local/apache2/conf/.htpasswd + Require valid-user + + # Restrict access to the /targets (dashboard) and /api/v1/targets (http) endpoints + # to the admin user + + ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/targets + ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/targets + AuthName "Prometheus" + AuthType Basic + AuthBasicProvider file + AuthUserFile /usr/local/apache2/conf/.htpasswd + Require valid-user + + + ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/api/v1/targets + ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/api/v1/targets + AuthName "Prometheus" + AuthType Basic + AuthBasicProvider file + AuthUserFile /usr/local/apache2/conf/.htpasswd + Require valid-user + + # Restrict access to the /api/v1/admin/tsdb/ endpoints (http) to the admin user. + # These endpoints are disabled by default, but are included here to ensure only + # an admin user has access to these endpoints when enabled + + ProxyPass http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/api/v1/admin/tsdb/ + ProxyPassReverse http://localhost:{{ tuple "monitoring" "internal" "api" . | include "helm-toolkit.endpoints.endpoint_port_lookup" }}/api/v1/admin/tsdb/ + AuthName "Prometheus" + AuthType Basic + AuthBasicProvider file + AuthUserFile /usr/local/apache2/conf/.htpasswd + Require valid-user + + SSLEngine On + SSLProxyEngine on + SSLCertificateFile /etc/prometheus/certs/tls.crt + SSLCertificateKeyFile /etc/prometheus/certs/tls.key + SSLProtocol all -SSLv3 -TLSv1 -TLSv1.1 + SSLCipherSuite ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256 + SSLHonorCipherOrder on + +manifests: + certificates: true +... diff --git a/values_overrides/rabbitmq/2023.1-ubuntu_focal.yaml b/values_overrides/rabbitmq/2023.1-ubuntu_focal.yaml new file mode 100644 index 0000000000..2a17e4f2d2 --- /dev/null +++ b/values_overrides/rabbitmq/2023.1-ubuntu_focal.yaml @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + prometheus_rabbitmq_exporter_helm_tests: docker.io/openstackhelm/heat:2023.1-ubuntu_focal + rabbitmq_init: docker.io/openstackhelm/heat:2023.1-ubuntu_focal +... diff --git a/values_overrides/rabbitmq/2023.1-ubuntu_jammy.yaml b/values_overrides/rabbitmq/2023.1-ubuntu_jammy.yaml new file mode 100644 index 0000000000..b3bd64cb4d --- /dev/null +++ b/values_overrides/rabbitmq/2023.1-ubuntu_jammy.yaml @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + prometheus_rabbitmq_exporter_helm_tests: docker.io/openstackhelm/heat:2023.1-ubuntu_jammy + rabbitmq_init: docker.io/openstackhelm/heat:2023.1-ubuntu_jammy +... diff --git a/values_overrides/rabbitmq/2023.2-ubuntu_jammy.yaml b/values_overrides/rabbitmq/2023.2-ubuntu_jammy.yaml new file mode 100644 index 0000000000..1b07b9bf41 --- /dev/null +++ b/values_overrides/rabbitmq/2023.2-ubuntu_jammy.yaml @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + prometheus_rabbitmq_exporter_helm_tests: docker.io/openstackhelm/heat:2023.2-ubuntu_jammy + rabbitmq_init: docker.io/openstackhelm/heat:2023.2-ubuntu_jammy +... diff --git a/values_overrides/rabbitmq/2024.1-ubuntu_jammy.yaml b/values_overrides/rabbitmq/2024.1-ubuntu_jammy.yaml new file mode 100644 index 0000000000..dbcac31562 --- /dev/null +++ b/values_overrides/rabbitmq/2024.1-ubuntu_jammy.yaml @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + prometheus_rabbitmq_exporter_helm_tests: docker.io/openstackhelm/heat:2024.1-ubuntu_jammy + rabbitmq_init: docker.io/openstackhelm/heat:2024.1-ubuntu_jammy +... diff --git a/values_overrides/rabbitmq/2024.2-ubuntu_jammy.yaml b/values_overrides/rabbitmq/2024.2-ubuntu_jammy.yaml new file mode 100644 index 0000000000..1ca5f68308 --- /dev/null +++ b/values_overrides/rabbitmq/2024.2-ubuntu_jammy.yaml @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + prometheus_rabbitmq_exporter_helm_tests: docker.io/openstackhelm/heat:2024.2-ubuntu_jammy + rabbitmq_init: docker.io/openstackhelm/heat:2024.2-ubuntu_jammy +... diff --git a/values_overrides/rabbitmq/apparmor.yaml b/values_overrides/rabbitmq/apparmor.yaml new file mode 100644 index 0000000000..f6ca7efdbb --- /dev/null +++ b/values_overrides/rabbitmq/apparmor.yaml @@ -0,0 +1,25 @@ +--- +pod: + mandatory_access_control: + type: apparmor + rabbitmq-cluster-wait: + init: runtime/default + rabbitmq-cookie: runtime/default + rabbitmq-rabbitmq-cluster-wait: runtime/default + rabbitmq: + init: runtime/default + rabbitmq-password: runtime/default + rabbitmq-cookie: runtime/default + rabbitmq-perms: runtime/default + rabbitmq: runtime/default + prometheus-rabbitmq-exporter: + init: runtime/default + rabbitmq-exporter: runtime/default + rabbitmq-rabbitmq-test: + rabbitmq-rabbitmq-test: runtime/default + init: runtime/default + +monitoring: + prometheus: + enabled: true +... diff --git a/values_overrides/rabbitmq/builtin-metrics.yaml b/values_overrides/rabbitmq/builtin-metrics.yaml new file mode 100644 index 0000000000..68a2773d77 --- /dev/null +++ b/values_overrides/rabbitmq/builtin-metrics.yaml @@ -0,0 +1,16 @@ +--- +# This enable Rabbitmq built-in prometheus plugin +conf: + enabled_plugins: + - rabbitmq_management + - rabbitmq_peer_discovery_k8s + - rabbitmq_prometheus + +manifests: + monitoring: + prometheus: + configmap_bin: false + deployment_exporter: false + service_exporter: false + network_policy_exporter: false +... diff --git a/values_overrides/rabbitmq/netpol.yaml b/values_overrides/rabbitmq/netpol.yaml new file mode 100644 index 0000000000..3c50a7d71c --- /dev/null +++ b/values_overrides/rabbitmq/netpol.yaml @@ -0,0 +1,105 @@ +--- +network_policy: + rabbitmq: + ingress: + - from: + - podSelector: + matchLabels: + application: keystone + - podSelector: + matchLabels: + application: heat + - podSelector: + matchLabels: + application: glance + - podSelector: + matchLabels: + application: cinder + - podSelector: + matchLabels: + application: aodh + - podSelector: + matchLabels: + application: barbican + - podSelector: + matchLabels: + application: ceilometer + - podSelector: + matchLabels: + application: designate + - podSelector: + matchLabels: + application: ironic + - podSelector: + matchLabels: + application: magnum + - podSelector: + matchLabels: + application: mistral + - podSelector: + matchLabels: + application: nova + - podSelector: + matchLabels: + application: neutron + - podSelector: + matchLabels: + application: senlin + - podSelector: + matchLabels: + application: placement + - podSelector: + matchLabels: + application: rabbitmq + - podSelector: + matchLabels: + application: prometheus_rabbitmq_exporter + ports: + # AMQP port + - protocol: TCP + port: 5672 + # HTTP API ports + - protocol: TCP + port: 15672 + - protocol: TCP + port: 80 + - from: + - podSelector: + matchLabels: + application: rabbitmq + ports: + # Clustering port AMQP + 20000 + - protocol: TCP + port: 25672 + # Erlang Port Mapper Daemon (epmd) + - protocol: TCP + port: 4369 + egress: + - to: + - podSelector: + matchLabels: + application: rabbitmq + ports: + # Erlang port mapper daemon (epmd) + - protocol: TCP + port: 4369 + # Rabbit clustering port AMQP + 20000 + - protocol: TCP + port: 25672 + # NOTE(lamt): Set by inet_dist_listen_{min/max}. Firewalls must + # permit traffic in this range to pass between clustered nodes. + # - protocol: TCP + # port: 35197 + - to: + - ipBlock: + cidr: %%%REPLACE_API_ADDR%%%/32 + ports: + - protocol: TCP + port: %%%REPLACE_API_PORT%%% + +manifests: + monitoring: + prometheus: + network_policy_exporter: true + network_policy: true +... diff --git a/values_overrides/rabbitmq/rabbitmq-exporter.yaml b/values_overrides/rabbitmq/rabbitmq-exporter.yaml new file mode 100644 index 0000000000..0adedca27e --- /dev/null +++ b/values_overrides/rabbitmq/rabbitmq-exporter.yaml @@ -0,0 +1,10 @@ +--- +# This enable external pod for rabbitmq-exporter +manifests: + monitoring: + prometheus: + configmap_bin: true + deployment_exporter: true + service_exporter: true + network_policy_exporter: false +... diff --git a/values_overrides/rabbitmq/tls.yaml b/values_overrides/rabbitmq/tls.yaml new file mode 100644 index 0000000000..b4c241903b --- /dev/null +++ b/values_overrides/rabbitmq/tls.yaml @@ -0,0 +1,30 @@ +--- +conf: + rabbitmq: + ssl_options: + cacertfile: "/etc/rabbitmq/certs/ca.crt" + certfile: "/etc/rabbitmq/certs/tls.crt" + keyfile: "/etc/rabbitmq/certs/tls.key" + verify: verify_peer + fail_if_no_peer_cert: false + management: + ssl: + cacertfile: "/etc/rabbitmq/certs/ca.crt" + certfile: "/etc/rabbitmq/certs/tls.crt" + keyfile: "/etc/rabbitmq/certs/tls.key" +endpoints: + oslo_messaging: + host_fqdn_override: + default: + tls: + secretName: rabbitmq-tls-direct + issuerRef: + name: ca-issuer + kind: ClusterIssuer + port: + https: + default: 15680 + public: 443 +manifests: + certificates: true +... diff --git a/values_overrides/rabbitmq/yoga-ubuntu_focal.yaml b/values_overrides/rabbitmq/yoga-ubuntu_focal.yaml new file mode 100644 index 0000000000..4f29dc4b69 --- /dev/null +++ b/values_overrides/rabbitmq/yoga-ubuntu_focal.yaml @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + prometheus_rabbitmq_exporter_helm_tests: docker.io/openstackhelm/heat:yoga-ubuntu_focal + rabbitmq_init: docker.io/openstackhelm/heat:yoga-ubuntu_focal +... diff --git a/values_overrides/rabbitmq/zed-ubuntu_focal.yaml b/values_overrides/rabbitmq/zed-ubuntu_focal.yaml new file mode 100644 index 0000000000..907d962a05 --- /dev/null +++ b/values_overrides/rabbitmq/zed-ubuntu_focal.yaml @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + prometheus_rabbitmq_exporter_helm_tests: docker.io/openstackhelm/heat:zed-ubuntu_focal + rabbitmq_init: docker.io/openstackhelm/heat:zed-ubuntu_focal +... diff --git a/values_overrides/rabbitmq/zed-ubuntu_jammy.yaml b/values_overrides/rabbitmq/zed-ubuntu_jammy.yaml new file mode 100644 index 0000000000..bdc21d64e4 --- /dev/null +++ b/values_overrides/rabbitmq/zed-ubuntu_jammy.yaml @@ -0,0 +1,18 @@ +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- +images: + tags: + prometheus_rabbitmq_exporter_helm_tests: docker.io/openstackhelm/heat:zed-ubuntu_jammy + rabbitmq_init: docker.io/openstackhelm/heat:zed-ubuntu_jammy +... diff --git a/zuul.d/infra_jobs.yaml b/zuul.d/infra_jobs.yaml new file mode 100644 index 0000000000..f1be1c21ba --- /dev/null +++ b/zuul.d/infra_jobs.yaml @@ -0,0 +1,434 @@ +--- +# Copyright 2018 SUSE LINUX GmbH. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- job: + name: openstack-helm-lint + run: playbooks/lint.yml + nodeset: openstack-helm-1node-ubuntu_jammy + # NOTE(aostapenko) Required if job is run against another project + required-projects: + - openstack/openstack-helm-infra + irrelevant-files: + - ^.*\.rst$ + - ^releasenotes/.*$ + +- job: + name: openstack-helm-lint-osh + parent: openstack-helm-lint + required-projects: + - openstack/openstack-helm + files: + - ^helm-toolkit/.*$ + vars: + lint_osh: true + +- job: + name: openstack-helm-infra-bandit + roles: + - zuul: openstack/openstack-helm-infra + - zuul: zuul/zuul-jobs + required-projects: + - openstack/openstack-helm + - openstack/openstack-helm-infra + files: + - ^.*\.py\.tpl$ + - ^.*\.py$ + - ^playbooks/osh-infra-bandit.yaml$ + pre-run: playbooks/prepare-hosts.yaml + post-run: playbooks/osh-infra-collect-logs.yaml + run: playbooks/osh-infra-bandit.yaml + vars: + helm_version: "v3.6.3" + bandit_version: "1.7.1" + +- job: + name: publish-openstack-helm-charts + parent: publish-openstack-artifacts + run: playbooks/build-chart.yaml + required-projects: + - openstack/openstack-helm-infra + post-run: playbooks/publish/post.yaml + +- job: + name: openstack-helm-infra-deploy + abstract: true + roles: + - zuul: openstack/openstack-helm-infra + - zuul: zuul/zuul-jobs + required-projects: + - openstack/openstack-helm + - openstack/openstack-helm-infra + - openstack/openstack-helm-plugin + irrelevant-files: + - ^.*\.rst$ + - ^doc/.*$ + - ^releasenotes/.*$ + timeout: 10800 + pre-run: + - playbooks/prepare-hosts.yaml + - playbooks/mount-volumes.yaml + - playbooks/inject-keys.yaml + post-run: playbooks/osh-infra-collect-logs.yaml + run: + - playbooks/deploy-env.yaml + - playbooks/run-scripts.yaml + vars: + extra_volume: + size: 80G + type: Linux + mount_point: /opt/ext_vol + docker: + root_path: "/opt/ext_vol/docker" + containerd: + root_path: "/opt/ext_vol/containerd" + kubeadm: + pod_network_cidr: "10.244.0.0/16" + service_cidr: "10.96.0.0/16" + osh_plugin_repo: "{{ zuul.project.src_dir }}/../openstack-helm-plugin" + loopback_setup: true + loopback_device: /dev/loop100 + loopback_image: "/opt/ext_vol/openstack-helm/ceph-loop.img" + ceph_osd_data_device: /dev/loop100 + kube_version_repo: "v1.31" + kube_version: "1.31.3-1.1" + calico_setup: true + calico_version: "v3.27.4" + cilium_setup: false + cilium_version: "1.16.0" + flannel_setup: false + flannel_version: v0.25.4 + metallb_setup: false + metallb_version: "0.13.12" + ingress_setup: true + helm_version: "v3.14.0" + crictl_version: "v1.30.1" + zuul_osh_infra_relative_path: ../openstack-helm-infra + gate_scripts_relative_path: ../openstack-helm-infra + run_helm_tests: "no" + +- job: + name: openstack-helm-infra-logging + parent: openstack-helm-infra-deploy + nodeset: openstack-helm-5nodes-ubuntu_jammy + vars: + osh_params: + openstack_release: "2024.1" + container_distro_name: ubuntu + container_distro_version: jammy + gate_scripts: + - ./tools/deployment/common/prepare-k8s.sh + - ./tools/deployment/common/infra-prepare-charts.sh + - ./tools/deployment/ceph/ceph-rook.sh + - ./tools/deployment/ceph/ceph-adapter-rook.sh + - ./tools/deployment/common/ldap.sh + - ./tools/deployment/logging/elasticsearch.sh + - ./tools/deployment/logging/fluentd.sh + - ./tools/deployment/logging/kibana.sh + - ./tools/gate/selenium/kibana-selenium.sh || true + +- job: + name: openstack-helm-infra-monitoring + parent: openstack-helm-infra-deploy + nodeset: openstack-helm-1node-ubuntu_jammy + vars: + osh_params: + openstack_release: "2024.1" + container_distro_name: ubuntu + container_distro_version: jammy + gate_scripts: + - ./tools/deployment/common/prepare-k8s.sh + - ./tools/deployment/common/infra-prepare-charts.sh + - ./tools/deployment/common/deploy-docker-registry.sh + - ./tools/deployment/common/nfs-provisioner.sh + - ./tools/deployment/common/ldap.sh + - ./tools/deployment/db/mariadb.sh + - ./tools/deployment/db/postgresql.sh + - ./tools/deployment/monitoring/prometheus.sh + - ./tools/deployment/monitoring/alertmanager.sh + - ./tools/deployment/monitoring/kube-state-metrics.sh + - ./tools/deployment/monitoring/node-problem-detector.sh + - ./tools/deployment/monitoring/node-exporter.sh + - ./tools/deployment/monitoring/process-exporter.sh + - ./tools/deployment/monitoring/openstack-exporter.sh + - ./tools/deployment/monitoring/blackbox-exporter.sh + - ./tools/deployment/monitoring/grafana.sh + - ./tools/deployment/monitoring/nagios.sh + - ./tools/gate/selenium/grafana-selenium.sh || true + - ./tools/gate/selenium/prometheus-selenium.sh || true + - ./tools/gate/selenium/nagios-selenium.sh || true + +- job: + name: openstack-helm-infra-metacontroller + parent: openstack-helm-infra-deploy + nodeset: openstack-helm-1node-ubuntu_jammy + vars: + osh_params: + container_distro_name: ubuntu + container_distro_version: jammy + feature_gates: apparmor + ingress_setup: false + gate_scripts: + - ./tools/deployment/common/prepare-k8s.sh + - ./tools/deployment/common/infra-prepare-charts.sh + - ./tools/deployment/common/metacontroller.sh + - ./tools/deployment/common/daemonjob-controller.sh + +- job: + name: openstack-helm-infra-mariadb-operator-2024-1-ubuntu_jammy + parent: openstack-helm-infra-deploy + nodeset: openstack-helm-3nodes-ubuntu_jammy + pre-run: + - playbooks/prepare-hosts.yaml + - playbooks/mount-volumes.yaml + vars: + osh_params: + openstack_release: "2024.1" + container_distro_name: ubuntu + container_distro_version: jammy + feature_gates: "ldap,prometheus,backups" + gate_scripts: + - ./tools/deployment/common/prepare-k8s.sh + - ./tools/deployment/common/infra-prepare-charts.sh + - ./tools/deployment/common/namespace-config.sh + - ./tools/deployment/ceph/ceph.sh + - ./tools/deployment/ceph/ceph-ns-activate.sh + - ./tools/deployment/common/setup-client.sh + - ./tools/deployment/common/rabbitmq.sh + - ./tools/deployment/common/memcached.sh + - ./tools/deployment/db/mariadb-operator-cluster.sh + - export NAMESPACE=openstack; ./tools/deployment/common/ldap.sh + - | + export OSH_EXTRA_HELM_ARGS="--set endpoints.oslo_db.hosts.default=mariadb-server-primary ${OSH_EXTRA_HELM_ARGS}" + ./tools/deployment/openstack/keystone.sh + - ./tools/deployment/db/mariadb-backup.sh + - ./tools/deployment/monitoring/mysql-exporter.sh + files: + - ^roles/.* + - ^mariadb-cluster/.* + - ^tools/.* + +- job: + name: openstack-helm-infra-compute-kit-dpdk-2024-1-ubuntu_jammy + description: | + Run the openstack-helm compute-kit job with DPDK enabled. + We use single node environment to run this job which means + that the job only tests that QEMU and OVS-DPDK are working + together. The job does not assume having specific DPDK hardware. + parent: openstack-helm-compute-kit + pre-run: + - playbooks/enable-hugepages.yaml + - playbooks/prepare-hosts.yaml + nodeset: openstack-helm-1node-32GB-ubuntu_jammy + vars: + gate_scripts_relative_path: ../openstack-helm + hugepages: + enabled: true + size: "2M" + number: 2048 + osh_params: + openstack_release: "2024.1" + container_distro_name: ubuntu + container_distro_version: jammy + feature_gates: dpdk + files: + - ^roles/.* + - ^openvswitch/.* + +- job: + name: openstack-helm-infra-compute-kit-ovn-2024-1-ubuntu_jammy + parent: openstack-helm-compute-kit-ovn-2024-1-ubuntu_jammy + files: + - ^helm-toolkit/.* + - ^roles/.* + - ^rabbitmq/.* + - ^mariadb/.* + - ^libvirt/.* + - ^ovn/.* + +- job: + name: openstack-helm-infra-compute-kit-2024-1-ubuntu_jammy + parent: openstack-helm-compute-kit-2024-1-ubuntu_jammy + files: + - ^helm-toolkit/.* + - ^roles/.* + - ^rabbitmq/.* + - ^mariadb/.* + - ^libvirt/.* + - ^memcached/.* + - ^openvswitch/.* + +- job: + name: openstack-helm-infra-compute-kit-2024-2-ubuntu_jammy + parent: openstack-helm-compute-kit-2024-2-ubuntu_jammy + files: + - ^helm-toolkit/.* + - ^roles/.* + - ^rabbitmq/.* + - ^mariadb/.* + - ^libvirt/.* + - ^memcached/.* + - ^openvswitch/.* + +- job: + name: openstack-helm-infra-keystone-cilium-2024-1-ubuntu_jammy + parent: openstack-helm-infra-deploy + nodeset: openstack-helm-3nodes-ubuntu_jammy + files: + - ^helm-toolkit/.* + - ^roles/.* + - ^rabbitmq/.* + - ^mariadb/.* + - ^memcached/.* + vars: + osh_params: + openstack_release: "2024.1" + container_distro_name: ubuntu + container_distro_version: jammy + calico_setup: false + cilium_setup: true + gate_scripts: + - ./tools/deployment/common/prepare-k8s.sh + - ./tools/deployment/common/infra-prepare-charts.sh + - ./tools/deployment/common/setup-client.sh + - | + export NAMESPACE=openstack + export OSH_INFRA_EXTRA_HELM_ARGS="--set pod.replicas.server=1 --set volume.enabled=false --set volume.use_local_path_for_single_pod_cluster.enabled=true ${OSH_INFRA_EXTRA_HELM_ARGS}" + export RUN_HELM_TESTS=no + ./tools/deployment/db/mariadb.sh + - ./tools/deployment/common/rabbitmq.sh + - ./tools/deployment/common/memcached.sh + - ./tools/deployment/openstack/keystone.sh + +- job: + name: openstack-helm-infra-keystone-flannel-2024-1-ubuntu_jammy + parent: openstack-helm-infra-deploy + nodeset: openstack-helm-3nodes-ubuntu_jammy + files: + - ^helm-toolkit/.* + - ^roles/.* + - ^rabbitmq/.* + - ^mariadb/.* + - ^memcached/.* + vars: + osh_params: + openstack_release: "2024.1" + container_distro_name: ubuntu + container_distro_version: jammy + calico_setup: false + flannel_setup: true + gate_scripts: + - ./tools/deployment/common/prepare-k8s.sh + - ./tools/deployment/common/infra-prepare-charts.sh + - ./tools/deployment/common/setup-client.sh + - | + export NAMESPACE=openstack + export OSH_INFRA_EXTRA_HELM_ARGS="--set pod.replicas.server=1 --set volume.enabled=false --set volume.use_local_path_for_single_pod_cluster.enabled=true ${OSH_INFRA_EXTRA_HELM_ARGS}" + export RUN_HELM_TESTS=no + ./tools/deployment/db/mariadb.sh + - ./tools/deployment/common/rabbitmq.sh + - ./tools/deployment/common/memcached.sh + - ./tools/deployment/openstack/keystone.sh + +- job: + name: openstack-helm-infra-cinder-2024-1-ubuntu_jammy + description: | + This job uses Rook for managing Ceph cluster. + The job is run on 5 nodes. + parent: openstack-helm-cinder-2024-1-ubuntu_jammy + nodeset: openstack-helm-5nodes-ubuntu_jammy + files: + - ^helm-toolkit/.* + - ^roles/.* + - ^ceph.* + - ^tools/deployment/ceph/ceph-rook\.sh$ + - ^tools/deployment/ceph/ceph-adapter-rook\.sh$ + +- job: + name: openstack-helm-infra-tls-2024-1-ubuntu_jammy + description: | + This job uses Rook for managing Ceph cluster. + parent: openstack-helm-tls-2024-1-ubuntu_jammy + # NOTE(kozhukalov): The job is not stable. We make it non-voting for now + # to unblock the gate. + voting: false + files: + - ^helm-toolkit/.* + - ^roles/.* + - ^rabbitmq/.* + - ^mariadb/.* + - ^memcached/.* + - ^libvrit/.* + - ^openvswitch/.* + +- job: + name: openstack-helm-infra-mariadb-ingress-2024-1-ubuntu_jammy + parent: openstack-helm-compute-kit-2024-1-ubuntu_jammy + vars: + osh_params: + feature_gates: "ingress-service" + files: + - ^helm-toolkit/.* + - ^roles/.* + - ^rabbitmq/.* + - ^mariadb/.* + - ^memcached/.* + - ^libvrit/.* + - ^openvswitch/.* + +- job: + name: openstack-helm-infra-ceph-migrate + description: | + This job is for testing the migration procedure from + a Ceph cluster managed by legacy OSH ceph* charts + to a Ceph cluster managed by Rook-Ceph operator. + parent: openstack-helm-infra-deploy + nodeset: openstack-helm-5nodes-ubuntu_jammy + timeout: 10800 + pre-run: + - playbooks/prepare-hosts.yaml + - playbooks/mount-volumes.yaml + - playbooks/inject-keys.yaml + files: + - ^helm-toolkit/.* + - ^roles/.* + - ^ceph.* + - ^tools/deployment/ceph/.* + vars: + osh_params: + openstack_release: "2024.1" + container_distro_name: ubuntu + container_distro_version: jammy + gate_scripts: + - ./tools/deployment/common/prepare-k8s.sh + - ./tools/deployment/common/infra-prepare-charts.sh + # Deploy Ceph cluster using legacy OSH charts + - ./tools/deployment/ceph/ceph_legacy.sh + # Deploy stateful applications + - | + export NAMESPACE=openstack + export MONITORING_HELM_ARGS=" " + export OSH_INFRA_EXTRA_HELM_ARGS="--set pod.replicas.server=1 ${OSH_INFRA_EXTRA_HELM_ARGS}" + export RUN_HELM_TESTS=no + ./tools/deployment/db/mariadb.sh + - | + export NAMESPACE=openstack + export VOLUME_HELM_ARGS=" " + ./tools/deployment/common/rabbitmq.sh + # Migrate legacy Ceph to Rook + - ./tools/deployment/ceph/migrate-before.sh + - ./tools/deployment/ceph/migrate-values.sh + - ./tools/deployment/ceph/migrate-to-rook-ceph.sh + - ./tools/deployment/ceph/migrate-after.sh +... diff --git a/zuul.d/infra_project.yaml b/zuul.d/infra_project.yaml new file mode 100644 index 0000000000..0daa66aa6c --- /dev/null +++ b/zuul.d/infra_project.yaml @@ -0,0 +1,53 @@ +--- +# Copyright 2018 SUSE LINUX GmbH. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +- project: + templates: + - release-notes-jobs-python3 + check: + jobs: + - openstack-helm-lint + - openstack-helm-lint-osh + - openstack-helm-infra-bandit + - openstack-helm-infra-logging + - openstack-helm-infra-monitoring + - openstack-helm-infra-metacontroller + - openstack-helm-infra-mariadb-operator-2024-1-ubuntu_jammy + - openstack-helm-infra-compute-kit-ovn-2024-1-ubuntu_jammy + - openstack-helm-infra-compute-kit-2024-1-ubuntu_jammy + - openstack-helm-infra-cinder-2024-1-ubuntu_jammy + - openstack-helm-infra-tls-2024-1-ubuntu_jammy + - openstack-helm-infra-compute-kit-dpdk-2024-1-ubuntu_jammy # 32GB node + - openstack-helm-infra-keystone-cilium-2024-1-ubuntu_jammy + - openstack-helm-infra-keystone-flannel-2024-1-ubuntu_jammy + - openstack-helm-infra-compute-kit-2024-2-ubuntu_jammy + gate: + jobs: + - openstack-helm-lint + - openstack-helm-lint-osh + - openstack-helm-infra-logging + - openstack-helm-infra-monitoring + - openstack-helm-infra-metacontroller + post: + jobs: + - publish-openstack-helm-charts + periodic: + jobs: + - publish-openstack-helm-charts + periodic-weekly: + jobs: + - openstack-helm-infra-ceph-migrate + +...