Generate CHANGELOG.md for charts

We use reno>=4.1.0 features to combine release notes from
a bunch of release notes files. Reno uses git tags
to figure out which release notes files to include
to a given release.

When building charts for deployment tests we skip
generating CHANGELOG.md files.

Change-Id: If908b9b6c8d0adae422deaefb25ba56ece701aa5

Makefile

@@ -12,11 +12,17 @@
 
 # It's necessary to set this because some environments don't link sh -> bash.
 SHELL := /bin/bash
-HELM  := helm
+HELM := helm
+PYTHON := python3
 TASK  := build
 HELM_DOCS := tools/helm-docs
 UNAME_OS := $(shell uname -s)
 UNAME_ARCH := $(shell uname -m)
+# We generate CHANGELOG.md files by default which
+# requires reno>=4.1.0 installed.
+# To skip generating it use the following:
+# make all SKIP_CHANGELOG=1
+SKIP_CHANGELOG ?= 0
 
 PKG_ARGS =
 ifdef VERSION
@@ -36,6 +42,9 @@ CHARTS := $(sort helm-toolkit $(CHART_DIRS))
 
 all: $(CHARTS)
 
+charts:
+	@echo $(CHART_DIRS)
+
 $(CHARTS):
 	@if [ -d $@ ]; then \
 		echo; \
@@ -61,7 +70,11 @@ init-%:
 lint-%: init-%
 	if [ -d $* ]; then $(HELM) lint $*; fi
 
-build-%: lint-%
+# reno required for changelog generation
+%/CHANGELOG.md:
+	if [ -d $* ]; then $(PYTHON) tools/changelog.py --charts $*; fi
+
+build-%: lint-% $(if $(filter-out 1,$(SKIP_CHANGELOG)),%/CHANGELOG.md)
 	if [ -d $* ]; then \
 		$(HELM) package $* --version $$(tools/chart_version.sh $* $(BASE_VERSION)) $(PKG_ARGS); \
 	fi

README.rst

@@ -82,3 +82,28 @@ Our repositories:
 * Building Openstack images framework `loci <https://opendev.org/openstack/loci.git>`_
 
 We welcome contributions in any form: code review, code changes, usage feedback, updating documentation.
+
+Release notes
+-------------
+
+We use `reno <https://opendev.org/openstack/reno.git>`_ for managing release notes. If you update
+a chart, please add a release note using the following command:
+
+.. code-block:: bash
+
+    reno new <chart_name>
+
+This will create a new release note file ``releasenotes/notes/<chart_name>-<sha>.yaml``. Fill in the
+necessary information and commit the release note file.
+
+If you update multiple charts in a single commit use the following command:
+
+.. code-block:: bash
+
+    reno new common
+
+This will create a new release note file ``releasenotes/notes/common-<sha>.yaml``. In this case you
+can add multiple chart specific sections in this release note file.
+
+When building tarballы, we will use the ``reno`` features to combine release notes from all files and
+generate  ``<chart_name>/CHANGELOG.md`` files.

releasenotes/config.yaml

@@ -28,10 +28,59 @@ sections:
   - [api, API Changes]
   - [security, Security Issues]
   - [fixes, Bug Fixes]
-  - [other, Other Changes]
 template: |
   ---
+  # To create a new release note related to a specific chart:
+  # reno new <chart_name>
+  #
+  # To create a new release note for a common change (when multiple charts
+  # are changed):
+  # reno new common
   <chart_name>:
-    - Short change description
+    - |
+      Describe changes here, or remove this section. This paragraph will appear in
+      the unnamed section of the <chart_name>/CHANGELOG.md for a given version.
+  features:
+    - |
+      List new features here, or remove this section. If this section is given
+      in the releasenotes/notes/<chart_name>-<sha>.yaml it will only appear in the
+      "New Features" section of the <chart_name>/CHANGELOG.md. If this section is
+      given in the releasenotes/notes/common-<sha>.yaml it will appear in the
+      CHANGELOG.md files of all charts.
+  issues:
+    - |
+      List known issues here, or remove this section. If this section is given
+      in the releasenotes/notes/<chart_name>-<sha>.yaml it will only appear in the
+      "Known Issues" section of the <chart_name>/CHANGELOG.md. If this section is
+      given in the releasenotes/notes/common-<sha>.yaml it will appear in the
+      CHANGELOG.md files of all charts.
+  upgrade:
+    - |
+      List upgrade notes here, or remove this section. If this section is given
+      in the releasenotes/notes/<chart_name>-<sha>.yaml it will only appear in the
+      "Upgrade Notes" section of the <chart_name>/CHANGELOG.md. If this section is
+      given in the releasenotes/notes/common-<sha>.yaml it will appear in the
+      CHANGELOG.md files of all charts.
+  api:
+    - |
+      List API changes here (e.g. values format), or remove this section. If this section is given
+      in the releasenotes/notes/<chart_name>-<sha>.yaml it will only appear in the
+      "API Changes" section of the <chart_name>/CHANGELOG.md. If this section is
+      given in the releasenotes/notes/common-<sha>.yaml it will appear in the
+      CHANGELOG.md files of all charts.
+  security:
+    - |
+      List security issues here, or remove this section. If this section is given
+      in the releasenotes/notes/<chart_name>-<sha>.yaml it will only appear in the
+      "Security Issues" section of the <chart_name>/CHANGELOG.md. If this section is
+      given in the releasenotes/notes/common-<sha>.yaml it will appear in the
+      CHANGELOG.md files of all charts.
+  fixes:
+    - |
+      List bug fixes here, or remove this section. If this section is given
+      in the releasenotes/notes/<chart_name>-<sha>.yaml it will only appear in the
+      "Bug Fixes" section of the <chart_name>/CHANGELOG.md. If this section is
+      given in the releasenotes/notes/common-<sha>.yaml it will appear in the
+      CHANGELOG.md files of all charts.
   ...
 ...

tools/changelog.py

@@ -0,0 +1,132 @@
+import argparse
+import os.path
+from collections import defaultdict
+
+from reno import config
+from reno import loader
+
+
+BEFORE_2024_2_0_NOTE = """Before 2024.2.0 all the OpenStack-Helm charts were versioned independently.
+Here we provide all the release notes for the chart for all versions before 2024.2.0.
+"""
+
+def _indent_for_list(text, prefix='  '):
+    lines = text.splitlines()
+    return '\n'.join([lines[0]] + [
+        prefix + l
+        for l in lines[1:]
+    ])
+
+
+def chart_reports(loader, config, versions_to_include, title=None, charts=None):
+    reports = defaultdict(list)
+
+    file_contents = {}
+    for version in versions_to_include:
+        for filename, sha in loader[version]:
+            body = loader.parse_note_file(filename, sha)
+            file_contents[filename] = body
+
+    for chart in charts:
+        if title:
+            reports[chart].append(f"# {title}")
+            reports[chart].append('')
+
+        for version in versions_to_include:
+            if '-' in version:
+                version_title = config.unreleased_version_title or version
+            else:
+                version_title = version
+
+            reports[chart].append(f"## {version_title}")
+            reports[chart].append('')
+
+            if version == "2024.2.0":
+                reports[chart].append(BEFORE_2024_2_0_NOTE)
+
+            if config.add_release_date:
+                reports[chart].append('Release Date: ' + loader.get_version_date(version))
+                reports[chart].append('')
+
+            notefiles = loader[version]
+
+            # Prepare not named section
+            # 1. Get all files named <chart>*.yaml
+            #    and get <chart> section from all these files
+            # 2. Get all files named common*.yaml and get <chart>
+            #    section from all these files
+            is_content = False
+            for fn, sha in notefiles:
+                if os.path.basename(fn).startswith(chart) or \
+                        os.path.basename(fn).startswith("common"):
+                    notes = file_contents[fn].get(chart, [])
+                    for n in notes:
+                        is_content = True
+                        reports[chart].append(f"- {_indent_for_list(n)}")
+
+            # Add new line after unnamed section if it is not empty
+            if is_content:
+                reports[chart].append("")
+
+            # Prepare named sections
+            # 1. Get all files named <chart>*.yaml
+            #    and get all sections from all these files except <chart>
+            # 2. Get all files named common*.yaml
+            #    and get all sections from all these files except <chart>
+            for section in config.sections:
+                is_content = False
+
+                # Skip chart specific sections
+                if section.name not in ["features", "isseus", "upgrade", "api", "security", "fixes"]:
+                    continue
+
+                for fn, sha in notefiles:
+                    if os.path.basename(fn).startswith(chart) or \
+                            os.path.basename(fn).startswith("common"):
+
+                        notes = file_contents[fn].get(section.name, [])
+
+                        if notes and not is_content:
+                            reports[chart].append(f"### {section.title}")
+                            reports[chart].append("")
+
+                        if notes:
+                            is_content = True
+                            for n in notes:
+                                reports[chart].append(f"- {_indent_for_list(n)}")
+
+                # Add new line after the section if it is not empty
+                if is_content:
+                    reports[chart].append("")
+
+        report = reports[chart]
+        reports[chart] = '\n'.join(report)
+
+    return reports
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--charts", nargs="+", default=[], help="Charts to generate release notes for")
+    args = parser.parse_args()
+
+    conf = config.Config(".", "releasenotes")
+
+    with loader.Loader(conf) as ldr:
+        versions = ldr.versions
+        reports = chart_reports(
+            ldr,
+            conf,
+            versions,
+            title="Release notes",
+            charts=args.charts,
+        )
+
+    for chart in reports:
+        with open(f"{chart}/CHANGELOG.md", "w") as f:
+            f.write(reports[chart])
+    return
+
+
+if __name__ == "__main__":
+    main()

tools/deployment/common/prepare-charts.sh

@@ -16,8 +16,8 @@ set -ex
 # Build all OSH Infra charts (necessary for Openstack deployment)
 (
     cd ${OSH_INFRA_PATH:-"../openstack-helm-infra"} &&
-    make all
+    make all SKIP_CHANGELOG=1
 )
 
 # Build all OSH charts
-make all
+make all SKIP_CHANGELOG=1