Prepare for upcoming merger with OSH

* Remove unnecessary scripts
* Sync identical scripts
* Update chart_version.sh so to make it identical with OSH
* Sync README.rst, tox.ini, .gitignore, Makefile
* Rename some files to prevent merge conflicts
* Sync releasenotes config

Change-Id: Ibfdebcb62a416fc1b15989a1fd89b897a783d8f4

.gitignore

@@ -43,6 +43,11 @@ output/*/index.html
 
 # Sphinx
 doc/build
+doc/source/chart/*
+!doc/source/chart/index.rst
+
+# installed tools
+tools/helm-docs
 
 # pbr generates these
 AUTHORS
@@ -75,6 +80,7 @@ releasenotes/build
 logs/
 tools/gate/local-overrides/
 playbooks/*.retry
+tmp/
 
 # Helm-toolkit dev
 helm-toolkit/templates/test.yaml

Makefile

@@ -13,8 +13,11 @@
 # It's necessary to set this because some environments don't link sh -> bash.
 SHELL := /bin/bash
 HELM := helm
-TASK := build
 PYTHON := python3
+TASK  := build
+HELM_DOCS := tools/helm-docs
+UNAME_OS := $(shell uname -s)
+UNAME_ARCH := $(shell uname -m)
 # We generate CHANGELOG.md files by default which
 # requires reno>=4.1.0 installed.
 # To skip generating it use the following:
@@ -39,10 +42,26 @@ CHARTS := $(sort helm-toolkit $(CHART_DIRS))
 
 all: $(CHARTS)
 
+charts:
+	@echo $(CHART_DIRS)
+
 $(CHARTS):
-	@echo
+	@if [ -d $@ ]; then \
-	@echo "===== Processing [$@] chart ====="
+		echo; \
-	@make $(TASK)-$@
+		echo "===== Processing [$@] chart ====="; \
+		make $(TASK)-$@; \
+	fi
+
+HELM_DOCS_VERSION ?= 1.14.2
+.PHONY: helm-docs ## Download helm-docs locally if necessary
+helm-docs: $(HELM_DOCS)
+$(HELM_DOCS):
+	{ \
+		curl -fsSL -o tools/helm-docs.tar.gz https://github.com/norwoodj/helm-docs/releases/download/v$(HELM_DOCS_VERSION)/helm-docs_$(HELM_DOCS_VERSION)_$(UNAME_OS)_$(UNAME_ARCH).tar.gz && \
+		tar -zxf tools/helm-docs.tar.gz -C tools helm-docs && \
+		rm -f tools/helm-docs.tar.gz && \
+		chmod +x tools/helm-docs; \
+	}
 
 init-%:
 	if [ -f $*/Makefile ]; then make -C $*; fi
@@ -60,20 +79,24 @@ build-%: lint-% $(if $(filter-out 1,$(SKIP_CHANGELOG)),%/CHANGELOG.md)
 		$(HELM) package $* --version $$(tools/chart_version.sh $* $(BASE_VERSION)) $(PKG_ARGS); \
 	fi
 
+# This is used exclusively with helm3 building in the gate to publish
+package-%: init-%
+	if [ -d $* ]; then $(HELM) package $* $(PKG_ARGS); fi
+
 clean:
-	@echo "Removed .b64, _partials.tpl, and _globals.tpl files"
+	@echo "Clean all build artifacts"
-	rm -f helm-toolkit/secrets/*.b64
+	rm -f */templates/_partials.tpl */templates/_globals.tpl
-	rm -f */templates/_partials.tpl
+	rm -f *tgz */charts/*tgz */requirements.lock
-	rm -f */templates/_globals.tpl
+	rm -rf */charts */tmpcharts
-	rm -f *tgz */charts/*tgz
-	rm -f */requirements.lock
-	-rm -rf */charts */tmpcharts
 
 pull-all-images:
-	@./tools/pull-images.sh
+	@./tools/deployment/common/pull-images.sh
 
 pull-images:
-	@./tools/pull-images.sh $(filter-out $@,$(MAKECMDGOALS))
+	@./tools/deployment/common/pull-images.sh $(filter-out $@,$(MAKECMDGOALS))
+
+dev-deploy:
+	@./tools/gate/devel/start.sh $(filter-out $@,$(MAKECMDGOALS))
 
 %:
 	@:

README.rst

@@ -1,27 +1,109 @@
-====================
+==============
-Openstack-Helm-Infra
+OpenStack-Helm
-====================
+==============
 
 Mission
 -------
 
-The goal of OpenStack-Helm-Infra is to provide charts for services or
+The goal of OpenStack-Helm is to provide a collection of Helm charts that
-integration of third-party solutions that are required to run OpenStack-Helm.
+simply, resiliently, and flexibly deploy OpenStack and related services
+on Kubernetes.
 
-For more information, please refer to the OpenStack-Helm repository_.
+Versions supported
+------------------
 
-.. _repository: https://github.com/openstack/openstack-helm
+The table below shows the combinations of the Openstack/Platform/Kubernetes versions
+that are tested and proved to work.
+
+.. list-table::
+   :widths: 30 30 30 30
+   :header-rows: 1
+
+   * - Openstack version
+     - Host OS
+     - Image OS
+     - Kubernetes version
+   * - 2023.2 (Bobcat)
+     - Ubuntu Jammy
+     - Ubuntu Jammy
+     - >=1.29,<=1.31
+   * - 2024.1 (Caracal)
+     - Ubuntu Jammy
+     - Ubuntu Jammy
+     - >=1.29,<=1.31
+   * - 2024.2 (Dalmatian)
+     - Ubuntu Jammy
+     - Ubuntu Jammy
+     - >=1.29,<=1.31
 
 Communication
 -------------
 
 * Join us on `IRC <irc://chat.oftc.net/openstack-helm>`_:
-  #openstack-helm on oftc
+  ``#openstack-helm`` on oftc
 * Join us on `Slack <https://kubernetes.slack.com/messages/C3WERB7DE/>`_
-  - #openstack-helm
+  (this is preferable way of communication): ``#openstack-helm``
+* Join us on `Openstack-discuss <https://lists.openstack.org/cgi-bin/mailman/listinfo/openstack-discuss>`_
+  mailing list (use subject prefix ``[openstack-helm]``)
 
-Contributing
+The list of Openstack-Helm core team members is available here
-------------
+`openstack-helm-core <https://review.opendev.org/#/admin/groups/1749,members>`_.
 
-We welcome contributions. Check out `this <CONTRIBUTING.rst>`_ document if
+Storyboard
-you would like to get involved.
+----------
+
+You found an issue and want to make sure we are aware of it? You can do so on our
+`Storyboard <https://storyboard.openstack.org/#!/project_group/64>`_.
+
+Bugs should be filed as stories in Storyboard, not GitHub.
+
+Please be as much specific as possible while describing an issue. Usually having
+more context in the bug description means less efforts for a developer to
+reproduce the bug and understand how to fix it.
+
+Also before filing a bug to the Openstack-Helm `Storyboard <https://storyboard.openstack.org/#!/project_group/64>`_
+please try to identify if the issue is indeed related to the deployment
+process and not to the deployable software.
+
+Other links
+-----------
+
+Our documentation is available `here <https://docs.openstack.org/openstack-helm/latest/>`_.
+
+This project is under active development. We encourage anyone interested in
+OpenStack-Helm to review the `code changes <https://review.opendev.org/q/(project:openstack/openstack-helm+OR+project:openstack/openstack-helm-infra+OR+project:openstack/openstack-helm-images+OR+project:openstack/loci)+AND+-is:abandoned>`_
+
+Our repositories:
+
+* OpenStack charts `openstack-helm <https://opendev.org/openstack/openstack-helm.git>`_
+* Infra charts `openstack-helm-infra <https://opendev.org/openstack/openstack-helm-infra.git>`_
+* OpenStack-Helm plugin `openstack-helm-plugin <https://opendev.org/openstack/openstack-helm-plugin.git>`_
+* Building images `openstack-helm-images <https://opendev.org/openstack/openstack-helm-images.git>`_
+* Building Openstack images framework `loci <https://opendev.org/openstack/loci.git>`_
+
+We welcome contributions in any form: code review, code changes, usage feedback, updating documentation.
+
+Release notes
+-------------
+
+We use `reno <https://opendev.org/openstack/reno.git>`_ for managing release notes. If you update
+a chart, please add a release note using the following command:
+
+.. code-block:: bash
+
+    reno new <chart_name>
+
+This will create a new release note file ``releasenotes/notes/<chart_name>-<sha>.yaml``. Fill in the
+necessary information and commit the release note file.
+
+If you update multiple charts in a single commit use the following command:
+
+.. code-block:: bash
+
+    reno new common
+
+This will create a new release note file ``releasenotes/notes/common-<sha>.yaml``. In this case you
+can add multiple chart specific sections in this release note file.
+
+When building tarballы, we will use the ``reno`` features to combine release notes from all files and
+generate  ``<chart_name>/CHANGELOG.md`` files.

releasenotes/config.yaml

@@ -3,13 +3,18 @@ branch: master
 collapse_pre_releases: false
 stop_at_branch_base: true
 sections:
+  - [aodh, aodh Chart]
+  - [barbican, barbican Chart]
   - [ca-issuer, ca-issuer Chart]
   - [calico, calico Chart]
+  - [ceilometer, ceilometer Chart]
   - [ceph-client, ceph-client Chart]
   - [ceph-mon, ceph-mon Chart]
   - [ceph-osd, ceph-osd Chart]
   - [ceph-provisioners, ceph-provisioners Chart]
+  - [cinder, cinder Chart]
   - [daemonjob-controller, daemonjob-controller Chart]
+  - [designate, designate Chart]
   - [elastic-apm-server, elastic-apm-server Chart]
   - [elastic-filebeat, elastic-filebeat Chart]
   - [elastic-metricbeat, elastic-metricbeat Chart]
@@ -20,10 +25,15 @@ sections:
   - [flannel, flannel Chart]
   - [fluentbit, fluentbit Chart]
   - [fluentd, fluentd Chart]
+  - [glance, glance Chart]
   - [gnocchi, gnocchi Chart]
   - [grafana, grafana Chart]
+  - [heat, heat Chart]
   - [helm-toolkit, helm-toolkit Chart]
+  - [horizon, horizon Chart]
   - [ingress, ingress Chart]
+  - [ironic, ironic Chart]
+  - [keystone, keystone Chart]
   - [kibana, kibana Chart]
   - [kube-dns, kube-dns Chart]
   - [kubernetes-keystone-webhook, kubernetes-keystone-webhook Chart]
@@ -32,28 +42,36 @@ sections:
   - [libvirt, libvirt Chart]
   - [local-storage, local-storage Chart]
   - [lockdown, lockdown Chart]
+  - [magnum, magnum Chart]
   - [mariadb, mariadb Chart]
   - [memcached, memcached Chart]
   - [metacontroller, metacontroller Chart]
+  - [mistral, mistral Chart]
   - [mongodb, mongodb Chart]
   - [nagios, nagios Chart]
   - [namespace-config, namespace-config Chart]
+  - [neutron, neutron Chart]
   - [nfs-provisioner, nfs-provisioner Chart]
+  - [nova, nova Chart]
+  - [octavia, octavia Chart]
   - [openvswitch, openvswitch Chart]
   - [ovn, ovn Chart]
+  - [placement, placement Chart]
   - [postgresql, postgresql Chart]
   - [powerdns, powerdns Chart]
-  - [prometheus, prometheus Chart]
   - [prometheus-alertmanager, prometheus-alertmanager Chart]
   - [prometheus-blackbox-exporter, prometheus-blackbox-exporter Chart]
   - [prometheus-kube-state-metrics, prometheus-kube-state-metrics Chart]
   - [prometheus-node-exporter, prometheus-node-exporter Chart]
   - [prometheus-openstack-exporter, prometheus-openstack-exporter Chart]
   - [prometheus-process-exporter, prometheus-process-exporter Chart]
+  - [prometheus, prometheus Chart]
   - [rabbitmq, rabbitmq Chart]
+  - [rally, rally Chart]
   - [redis, redis Chart]
   - [registry, registry Chart]
   - [shaker, shaker Chart]
+  - [tempest, tempest Chart]
   - [features, New Features]
   - [issues, Known Issues]
   - [upgrade, Upgrade Notes]

releasenotes/requirements.txt

@@ -2,9 +2,6 @@
 # of appearance. Changing the order has an impact on the overall integration
 # process, which may cause wedges in the gate later.
 
-# Versions from TOX_CONSTRAINTS_FILE are used
+sphinx>=2.0.0,!=2.1.0 # BSD
-# TOX_CONSTRAINTS_FILE defaulted to https://opendev.org/openstack/requirements/raw/branch/master/upper-constraints.txt
+openstackdocstheme>=2.2.1 # Apache-2.0
-sphinx
+reno>=3.1.0 # Apache-2.0
-sphinxcontrib-blockdiag
-openstackdocstheme
-reno

tools/chart_version.sh

@@ -20,6 +20,13 @@ else
     # if there is no tag $BASE_VERSION, then we count the number of commits since the beginning
     PATCH=$(git log --oneline $CHART_DIR | wc -l)
 fi
-OSH_INFRA_COMMIT_SHA=$(git rev-parse --short HEAD);
 
-echo "${MAJOR}.${MINOR}.${PATCH}+${OSH_INFRA_COMMIT_SHA}"
+COMMIT_SHA=$(git rev-parse --short HEAD);
+OSH_INFRA_COMMIT_SHA=$(cd ../openstack-helm-infra; git rev-parse --short HEAD);
+if [[ ${COMMIT_SHA} = ${OSH_INFRA_COMMIT_SHA} ]]; then
+    BUILD_META=${COMMIT_SHA}
+else
+    BUILD_META=${COMMIT_SHA}-${OSH_INFRA_COMMIT_SHA}
+fi
+
+echo "${MAJOR}.${MINOR}.${PATCH}+${BUILD_META}"

tools/deployment/common/cert-manager.sh

@@ -1,5 +0,0 @@
-#!/bin/bash
-
-set -eux
-
-./${OSH_PATH}tools/scripts/tls/cert-manager.sh

tools/deployment/common/ingress.sh

@@ -1,80 +0,0 @@
-#!/bin/bash
-
-#    Licensed under the Apache License, Version 2.0 (the "License"); you may
-#    not use this file except in compliance with the License. You may obtain
-#    a copy of the License at
-#
-#         http://www.apache.org/licenses/LICENSE-2.0
-#
-#    Unless required by applicable law or agreed to in writing, software
-#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
-#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
-#    License for the specific language governing permissions and limitations
-#    under the License.
-
-set -xe
-
-: ${HELM_INGRESS_NGINX_VERSION:="4.11.1"}
-
-helm repo add ingress-nginx https://kubernetes.github.io/ingress-nginx
-
-#NOTE: Deploy cluster ingress
-helm upgrade --install ingress-nginx-cluster ingress-nginx/ingress-nginx \
-  --version ${HELM_INGRESS_NGINX_VERSION} \
-  --namespace=kube-system \
-  --set controller.admissionWebhooks.enabled="false" \
-  --set controller.kind=DaemonSet \
-  --set controller.service.type=ClusterIP \
-  --set controller.scope.enabled="false" \
-  --set controller.hostNetwork="true" \
-  --set controller.ingressClassResource.name=nginx-cluster \
-  --set controller.ingressClassResource.controllerValue="k8s.io/ingress-nginx-cluster" \
-  --set controller.ingressClassResource.default="true" \
-  --set controller.ingressClass=nginx-cluster \
-  --set controller.labels.app=ingress-api
-
-#NOTE: Wait for deploy
-helm osh wait-for-pods kube-system
-
-#NOTE: Deploy namespace ingress
-helm upgrade --install ingress-nginx-openstack ingress-nginx/ingress-nginx \
-  --version ${HELM_INGRESS_NGINX_VERSION} \
-  --namespace=openstack \
-  --set controller.admissionWebhooks.enabled="false" \
-  --set controller.scope.enabled="true" \
-  --set controller.service.enabled="false" \
-  --set controller.ingressClassResource.name=nginx \
-  --set controller.ingressClassResource.controllerValue="k8s.io/ingress-nginx-openstack" \
-  --set controller.ingressClass=nginx \
-  --set controller.labels.app=ingress-api
-
-#NOTE: Wait for deploy
-helm osh wait-for-pods openstack
-
-helm upgrade --install ingress-nginx-ceph ingress-nginx/ingress-nginx \
-  --version ${HELM_INGRESS_NGINX_VERSION} \
-  --namespace=ceph \
-  --set controller.admissionWebhooks.enabled="false" \
-  --set controller.scope.enabled="true" \
-  --set controller.service.enabled="false" \
-  --set controller.ingressClassResource.name=nginx-ceph \
-  --set controller.ingressClassResource.controllerValue="k8s.io/ingress-nginx-ceph" \
-  --set controller.ingressClass=nginx-ceph \
-  --set controller.labels.app=ingress-api
-
-#NOTE: Wait for deploy
-helm osh wait-for-pods ceph
-
-helm upgrade --install ingress-nginx-osh-infra ingress-nginx/ingress-nginx \
-  --version ${HELM_INGRESS_NGINX_VERSION} \
-  --namespace=osh-infra \
-  --set controller.admissionWebhooks.enabled="false" \
-  --set controller.scope.enabled="true" \
-  --set controller.service.enabled="false" \
-  --set controller.ingressClassResource.name=nginx-osh-infra \
-  --set controller.ingressClassResource.controllerValue="k8s.io/ingress-nginx-osh-infra" \
-  --set controller.ingressClass=nginx-osh-infra \
-  --set controller.labels.app=ingress-api
-
-#NOTE: Wait for deploy
-helm osh wait-for-pods osh-infra

tools/deployment/common/prepare-k8s.sh

@@ -31,7 +31,9 @@ kubectl label --overwrite nodes --all ceph-mgr=enabled
 # and we don't need L2 overlay (will be implemented later).
 kubectl label --overwrite nodes -l "node-role.kubernetes.io/control-plane" l3-agent=enabled
 
-for NAMESPACE in ceph mariadb-operator openstack osh-infra; do
+kubectl label --overwrite nodes -l "node-role.kubernetes.io/control-plane" openstack-network-node=enabled
+
+for NAMESPACE in ceph openstack osh-infra; do
 tee /tmp/${NAMESPACE}-ns.yaml << EOF
 apiVersion: v1
 kind: Namespace

tools/deployment/common/pull-images.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+set -x
+
+if [ "x$1" == "x" ]; then
+  CHART_DIRS="$(echo ./*/)"
+else
+  CHART_DIRS="$(echo ./$1/)"
+fi
+
+for CHART_DIR in ${CHART_DIRS} ; do
+  if [ -e ${CHART_DIR}values.yaml ]; then
+    for IMAGE in $(cat ${CHART_DIR}values.yaml | yq '.images.tags | map(.) | join(" ")' | tr -d '"'); do
+      sudo docker inspect $IMAGE >/dev/null|| sudo docker pull $IMAGE
+    done
+  fi
+done

tools/deployment/common/setup-client.sh

@@ -59,6 +59,6 @@ sudo docker run \\
     -v /etc/openstack-helm:/etc/openstack-helm \\
     -e OS_CLOUD=\${OS_CLOUD} \\
     \${OPENSTACK_CLIENT_CONTAINER_EXTRA_ARGS} \\
-    docker.io/openstackhelm/openstack-client:\${OPENSTACK_RELEASE:-2023.2} openstack "\${args[@]}"
+    quay.io/airshipit/openstack-client:\${OPENSTACK_RELEASE:-2024.2} openstack "\${args[@]}"
 EOF
 sudo chmod +x /usr/local/bin/openstack

tools/deployment/db/mariadb-operator-cluster.sh

@@ -19,7 +19,7 @@ set -xe
 
 # install mariadb-operator
 helm repo add mariadb-operator https://mariadb-operator.github.io/mariadb-operator
-helm upgrade --install mariadb-operator mariadb-operator/mariadb-operator --version ${MARIADB_OPERATOR_RELEASE} -n mariadb-operator
+helm upgrade --install --create-namespace mariadb-operator mariadb-operator/mariadb-operator --version ${MARIADB_OPERATOR_RELEASE} -n mariadb-operator
 
 #NOTE: Wait for deploy
 helm osh wait-for-pods mariadb-operator

tox.ini

@@ -15,12 +15,14 @@ commands = {posargs}
 
 [testenv:docs]
 deps =
-    -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
+  -c{env:TOX_CONSTRAINTS_FILE:https://releases.openstack.org/constraints/upper/master}
-    -r{toxinidir}/doc/requirements.txt
+  -r{toxinidir}/doc/requirements.txt
 commands =
   rm -rf doc/build
-  sphinx-build -W -b html doc/source doc/build/html
+  make helm-docs
+  sphinx-build -W --keep-going -b html -j auto doc/source doc/build/html
 allowlist_externals =
+  make
   rm
 
 [testenv:pdf-docs]
@@ -31,6 +33,7 @@ allowlist_externals =
   rm
 commands =
   rm -rf doc/build/pdf
+  make helm-docs
   sphinx-build -W --keep-going -b latex -j auto doc/source doc/build/pdf
   make -C doc/build/pdf
 
@@ -40,6 +43,9 @@ deps =
   yamllint
 commands =
   rm -rf .yamllint
+  bash -c 'if [ ! -d ../openstack-helm-infra ]; then \
+      git clone https://opendev.org/openstack/openstack-helm-infra ../openstack-helm-infra; \
+  fi'
   bash ../openstack-helm-infra/tools/gate/lint.sh
 allowlist_externals =
   rm

zuul.d/infra_jobs.yaml

@@ -129,7 +129,7 @@
         container_distro_version: jammy
       gate_scripts:
         - ./tools/deployment/common/prepare-k8s.sh
-        - ./tools/deployment/common/prepare-charts.sh
+        - ./tools/deployment/common/infra-prepare-charts.sh
         - ./tools/deployment/ceph/ceph-rook.sh
         - ./tools/deployment/ceph/ceph-adapter-rook.sh
         - ./tools/deployment/common/ldap.sh
@@ -149,7 +149,7 @@
         container_distro_version: jammy
       gate_scripts:
         - ./tools/deployment/common/prepare-k8s.sh
-        - ./tools/deployment/common/prepare-charts.sh
+        - ./tools/deployment/common/infra-prepare-charts.sh
         - ./tools/deployment/common/deploy-docker-registry.sh
         - ./tools/deployment/common/nfs-provisioner.sh
         - ./tools/deployment/common/ldap.sh
@@ -181,7 +181,7 @@
       ingress_setup: false
       gate_scripts:
         - ./tools/deployment/common/prepare-k8s.sh
-        - ./tools/deployment/common/prepare-charts.sh
+        - ./tools/deployment/common/infra-prepare-charts.sh
         - ./tools/deployment/common/metacontroller.sh
         - ./tools/deployment/common/daemonjob-controller.sh
 
@@ -200,7 +200,7 @@
         feature_gates: "ldap,prometheus,backups"
       gate_scripts:
         - ./tools/deployment/common/prepare-k8s.sh
-        - ./tools/deployment/common/prepare-charts.sh
+        - ./tools/deployment/common/infra-prepare-charts.sh
         - ./tools/deployment/common/namespace-config.sh
         - ./tools/deployment/ceph/ceph.sh
         - ./tools/deployment/ceph/ceph-ns-activate.sh
@@ -300,7 +300,7 @@
       cilium_setup: true
       gate_scripts:
         - ./tools/deployment/common/prepare-k8s.sh
-        - ./tools/deployment/common/prepare-charts.sh
+        - ./tools/deployment/common/infra-prepare-charts.sh
         - ./tools/deployment/common/setup-client.sh
         - |
           export NAMESPACE=openstack
@@ -330,7 +330,7 @@
       flannel_setup: true
       gate_scripts:
         - ./tools/deployment/common/prepare-k8s.sh
-        - ./tools/deployment/common/prepare-charts.sh
+        - ./tools/deployment/common/infra-prepare-charts.sh
         - ./tools/deployment/common/setup-client.sh
         - |
           export NAMESPACE=openstack
@@ -412,7 +412,7 @@
         container_distro_version: jammy
       gate_scripts:
         - ./tools/deployment/common/prepare-k8s.sh
-        - ./tools/deployment/common/prepare-charts.sh
+        - ./tools/deployment/common/infra-prepare-charts.sh
         # Deploy Ceph cluster using legacy OSH charts
         - ./tools/deployment/ceph/ceph_legacy.sh
         # Deploy stateful applications