Add a Helm chart collator

This adds a Dockerfile for an image which works as a Helm chart
repository. Charts can be pulled from either pre-existing
Helm repos or from git repos.

Change-Id: I860394eea3c322f2b142ea00dd7cc0a6916c34d5

helm-chart-collator/Dockerfile

@@ -0,0 +1,36 @@
+FROM ubuntu:20.04 as chart-collator
+
+SHELL ["bash", "-exc"]
+ENV DEBIAN_FRONTEND noninteractive
+
+# Update distro and install ansible and git
+RUN apt-get update && \
+    apt-get dist-upgrade -y && \
+    apt-get install -y --no-install-recommends \
+        python3-minimal \
+        python3-pip \
+        python3-apt \
+        python3-setuptools \
+        openssh-client \
+        jq \
+        git && \
+    pip3 install --upgrade wheel && \
+    pip3 install --upgrade ansible && \
+    pip3 install --upgrade jmespath && \
+    pip3 install --upgrade yq && \
+    rm -rf /var/lib/apt/lists/*
+
+COPY playbooks /opt/playbooks
+
+ARG CHARTS=""
+RUN ansible-playbook -v /opt/playbooks/create_repository.yaml \
+  -i /opt/playbooks/inventory.yaml \
+  --extra-vars "CHARTS=$CHARTS"
+
+FROM chartmuseum/chartmuseum:latest
+
+COPY --from=chart-collator /charts /charts
+
+ENTRYPOINT /chartmuseum --debug --port=8080 \
+    --storage="local" \
+    --storage-local-rootdir=/charts

helm-chart-collator/Makefile

@@ -0,0 +1,134 @@
+# Copyright 2018 AT&T Intellectual Property.  All other rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+SHELL             := /bin/bash
+BUILD_DIR         ?= build
+PUSH_IMAGE        ?= false
+IMAGE_ID          ?= none
+COMMIT            ?= $(shell git rev-parse HEAD)
+LABEL             ?= org.airshipit.build=community
+IMAGE_NAME        ?= collator
+DOCKER_REGISTRY   ?= quay.io
+IMAGE_PREFIX      ?= airshipit
+IMAGE_TAG         ?= latest
+DISTRO            ?= debian_stable
+IMAGE             := ${DOCKER_REGISTRY}/${IMAGE_PREFIX}/${IMAGE_NAME}:${IMAGE_TAG}-${DISTRO}
+SH_TO_CHECK       := $(wildcard files/*.sh )
+PROXY             ?= http://proxy.foo.com:8000
+NO_PROXY          ?= localhost,127.0.0.1,.svc.cluster.local
+USE_PROXY         ?= false
+
+all: lint images
+
+check-docker:
+	@if [ -z $$(which docker) ]; then \
+	  echo "Missing \`docker\` client which is required for development"; \
+	  exit 2; \
+	fi
+
+images: check-docker build_collator
+
+docs: clean build_docs
+
+build_docs:
+	echo TODO
+
+run_images: run_collator
+
+run_collator: $(BUILD_DIR)/output-metadata.yaml
+	echo OK
+	#TODO consistance test
+
+$(BUILD_DIR)/output-metadata.yaml: $(BUILD_DIR)/image_id
+ifeq ($(USE_PROXY), true)
+docker run --rm -it -p 8080:8080 tester
+	docker run \
+	--rm  \
+	-e http_proxy=$(PROXY) \
+	-e https_proxy=$(PROXY) \
+	-e HTTP_PROXY=$(PROXY) \
+	-e HTTPS_PROXY=$(PROXY) \
+	-e no_proxy=$(NO_PROXY) \
+	-e NO_PROXY=$(NO_PROXY) \
+	$(shell cat $(BUILD_DIR)/image_id)
+else
+	docker run \
+	--rm  \
+	$(shell cat $(BUILD_DIR)/image_id)
+endif
+
+$(BUILD_DIR)/image_id: build_collator
+
+build_collator:
+	mkdir -p $(BUILD_DIR)
+ifeq ($(IMAGE_ID), none)
+ifeq ($(USE_PROXY), true)
+	docker build . \
+	--tag $(IMAGE) \
+	--label $(LABEL) \
+	--label "org.opencontainers.image.revision=$(COMMIT)" \
+	--label "org.opencontainers.image.created=\
+	$(shell date --rfc-3339=seconds --utc)" \
+	--label "org.opencontainers.image.title=$(IMAGE_NAME)" \
+	--build-arg http_proxy=$(PROXY) \
+	--build-arg https_proxy=$(PROXY) \
+	--build-arg HTTP_PROXY=$(PROXY) \
+	--build-arg HTTPS_PROXY=$(PROXY) \
+	--build-arg no_proxy=$(NO_PROXY) \
+	--build-arg NO_PROXY=$(NO_PROXY) \
+	--build-arg GIT_COMMIT=$(COMMIT)
+else
+	docker build . \
+	--tag $(IMAGE) \
+	--label $(LABEL) \
+	--label "org.opencontainers.image.revision=$(COMMIT)" \
+	--label "org.opencontainers.image.created=\
+	$(shell date --rfc-3339=seconds --utc)" \
+	--label "org.opencontainers.image.title=$(IMAGE_NAME)" \
+	--build-arg GIT_COMMIT=$(COMMIT)
+endif
+	echo $(shell docker images -q $(IMAGE)) > $(BUILD_DIR)/image_id
+else
+	echo $(IMAGE_ID) > $(BUILD_DIR)/image_id
+endif
+ifeq ($(PUSH_IMAGE), true)
+	docker push $(IMAGE)
+endif
+
+clean:
+ifeq ($(IMAGE_ID), none)
+	if [[ -s $(BUILD_DIR)/image_id ]]; \
+	then \
+	docker rmi $$(cat $(BUILD_DIR)/image_id); \
+	fi
+endif
+	rm -rf $(BUILD_DIR)
+
+# style checks
+lint: test-shellcheck
+	echo "TODO"
+
+tests: lint unit_tests
+
+test-shellcheck: $(SH_TO_CHECK)
+
+unit_tests:
+	echo TODO
+
+$(SH_TO_CHECK):
+	docker run --rm -v $(shell pwd):/mnt \
+	nlknguyen/alpine-shellcheck -x /mnt/$(@)
+
+.PHONY: test clean $(SH_TO_CHECK) test-shellcheck tests lint build_collator \
+  run_collator run_images all build_docs docs check-docker images

helm-chart-collator/README.md

@@ -0,0 +1,99 @@
+# Helm Chart Collator
+
+The Helm Chart Collator is used to create a Helm Chart Repository served from a Docker
+image via Chartmuseum. It allows a developer to request charts to be pulled from various
+locations and packaged into the resulting Docker image, which can then be used as a
+portable Helm Repository.
+
+## Setup
+
+Charts can be sourced from various locations. Each entry must be recorded in a
+user-defined file before building the image. When the list of charts has been created,
+the `build-image.sh` script can be used to create the image via the command:
+
+```
+./build-image.sh $CHARTSFILE
+```
+
+### Charts from Helm Repos
+
+To pull a chart a from pre-existing Helm Repos by listing them under the `helm_repos`
+heading. Each listing must include the following:
+
+* `repo`: The name of the Helm Repo to add (e.g. `stable`)
+* `url`: The URL where the Helm Repo is hosted (e.g. `https://kubernetes-charts.storage.googleapis.com`)
+* `name`: The name of the desired chart (e.g. `mariadb`)
+* `version`: The version of the desired chart (e.g. `7.3.14`)
+
+
+### Charts from Git Repos
+
+A Chart can be pulled and packaged from a git repo by listing it under the `git_repos`
+heading. Listings must include:
+
+* `name`: The name of the repository (e.g. `openstack-helm`). Note that this is simply
+  used for caching during the cloning process.
+* `path`: The path to the desired chart within the repo (e.g. `keystone`)
+* `url`: The URL where the git repo is hosted (e.g. `https://github.com/openstack/openstack-helm`)
+* `sha`: The SHA-1 of the commit from which the chart should be pulled (e.g. `30c9f003d227b799c636458dea161e24d5823c33`). (default: `HEAD`).
+* `refspec`: The refspec associated with the `sha`. This is only required if the `sha`
+  can't be reached from the default (e.g. `refs/heads/master`)
+* `chart_version`: The version to package the chart with (e.g. `1.2.3`)
+
+If a chart in a git repo specifies dependencies which are not accessible, the
+dependencies must also be listed under the `dependencies` heading. Dependencies have the
+same fields as git repos.
+
+### Charts from Tarballs
+
+A chart can be downloaded by listing it under the `tarred_charts` header. They
+require the following:
+
+* `url`: The URL from which the chart can be downloaded
+
+## Example
+
+The following shows an example file for including various helm charts:
+* rook-ceph as a tarball from a git repo
+* mariadb from the helm stable repo
+* rook-ceph from the rook repo
+* prometheus from the helm/charts git repo
+* keystone from the openstack-helm git repo
+  * The helm-toolkit is also pulled, since it is a dependency of keystone
+
+```
+tarred_charts:
+  - url: https://github.com/project-azorian/rook-ceph-aio/raw/master/rook-ceph-aio/charts/rook-ceph-0.0.1.tgz
+helm_repos:
+  - repo: stable
+    url: https://kubernetes-charts.storage.googleapis.com
+    name: mariadb
+    version: 7.3.14
+  - repo: rook-release
+    url: https://charts.rook.io/release
+    name: rook-ceph
+    version: v1.3.6
+git_repos:
+  - name: helm-stable
+    path: stable/prometheus
+    url: https://github.com/helm/charts
+    sha: 79066e1f0f5ce735aeb4783f2adf4b85992d15de
+    # Note: refspec is only needed when if the given sha is not already available
+    refspec: refs/heads/master
+  - name: openstack-helm
+    path: keystone
+    url: https://github.com/openstack/openstack-helm
+    sha: 30c9f003d227b799c636458dea161e24d5823c33
+    chart_version: 1.2.3
+    dependencies:
+      - name: openstack-helm-infra
+        path: helm-toolkit
+        url: https://github.com/openstack/openstack-helm-infra
+        sha: b1e66fd308b6bc9df090aebb5b3807a0df2d87dd
+```
+
+Once this file has been created, the image can be built with the following:
+
+```
+./build-image.sh charts.yaml
+```

helm-chart-collator/build-image.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+if [[ $# != 1 ]]; then
+  printf "usage: ./%s <filename>\n" "$0"
+  exit 1
+fi
+
+docker build . -t helm-chart-collator --build-arg "CHARTS=\"$(cat "$1")\""

helm-chart-collator/examples/charts.yaml

@@ -0,0 +1,28 @@
+tarred_charts:
+  - url: https://github.com/project-azorian/rook-ceph-aio/raw/master/rook-ceph-aio/charts/rook-ceph-0.0.1.tgz
+helm_repos:
+  - repo: stable
+    url: https://kubernetes-charts.storage.googleapis.com
+    name: mariadb
+    version: 7.3.14
+  - repo: rook-release
+    url: https://charts.rook.io/release
+    name: rook-ceph
+    version: v1.3.6
+git_repos:
+  - name: helm-stable
+    path: stable/prometheus
+    url: https://github.com/helm/charts
+    sha: 79066e1f0f5ce735aeb4783f2adf4b85992d15de
+    # Note: refspec is only needed when if the given sha is not already available
+    refspec: refs/heads/master
+  - name: openstack-helm
+    path: keystone
+    url: https://github.com/openstack/openstack-helm
+    sha: 30c9f003d227b799c636458dea161e24d5823c33
+    chart_version: 1.2.3
+    dependencies:
+      - name: openstack-helm-infra
+        path: helm-toolkit
+        url: https://github.com/openstack/openstack-helm-infra
+        sha: b1e66fd308b6bc9df090aebb5b3807a0df2d87dd

helm-chart-collator/playbooks/create_repository.yaml

@@ -0,0 +1,33 @@
+---
+- hosts: all
+  tasks:
+
+  - name: create charts directory
+    file:
+      path: /charts
+      state: directory
+
+  - include_role:
+      name: ensure_helm
+
+  - include_role:
+      name: install_helm_repo_charts
+    loop: "{{ CHARTS | from_yaml | json_query('helm_repos[*]') | default([], true) }}"
+    loop_control:
+      loop_var: chart
+
+  - include_role:
+      name: install_git_repo_charts
+    loop: "{{ CHARTS | from_yaml | json_query('git_repos[*]') | default([], true) }}"
+    loop_control:
+      loop_var: chart
+
+  - include_role:
+      name: install_tarred_charts
+    loop: "{{ CHARTS | from_yaml | json_query('tarred_charts[*]') | default([], true) }}"
+    loop_control:
+      loop_var: chart
+
+  - name: create index.yaml
+    shell:
+      cmd: helm repo index /charts > /charts/index.yaml

helm-chart-collator/playbooks/inventory.yaml

@@ -0,0 +1,5 @@
+all:
+  hosts:
+    localhost:
+      ansible_connection: local
+      ansible_python_interpreter: /usr/bin/python3

helm-chart-collator/playbooks/roles/ensure_helm/tasks/main.yaml

@@ -0,0 +1,12 @@
+---
+- name: download and unarchive helm
+  unarchive:
+    src: https://get.helm.sh/helm-v3.2.3-linux-amd64.tar.gz
+    remote_src: yes
+    dest: /
+
+- name: move helm into the path
+  command: mv /linux-amd64/helm /bin/helm
+
+- name: assert helm is executable
+  command: helm version

helm-chart-collator/playbooks/roles/install_git_repo_charts/tasks/dependencies.yaml

@@ -0,0 +1,18 @@
+---
+- name: clone dependency repo
+  git:
+    dest: /tmp/{{ chart_dependency["name"] }}
+    repo: "{{ chart_dependency['url'] }}"
+    version: "{{ chart_dependency['sha'] | default('HEAD') }}"
+    refspec: "{{ chart_dependency['refspec'] | default('refs/heads/master') }}"
+    depth: 1
+
+- name: ensure the parent's charts directory exists
+  file:
+    path: /tmp/{{ chart["name"] }}/{{ chart["path"] }}/charts
+    state: directory
+
+- name: move dependency into parent's charts directory
+  local_action: >-
+    command cp -r /tmp/{{ chart_dependency["name"] }}/{{ chart_dependency["path"] }}
+                  /tmp/{{ chart["name"] }}/{{ chart["path"] }}/charts

helm-chart-collator/playbooks/roles/install_git_repo_charts/tasks/main.yaml

@@ -0,0 +1,30 @@
+---
+- name: clone repos
+  git:
+    dest: /tmp/{{ chart['name'] }}
+    repo: "{{ chart['url'] }}"
+    version: "{{ chart['sha'] | default('HEAD') }}"
+    refspec: "{{ chart['refspec'] | default('refs/heads/master') }}"
+    depth: 1
+
+- include_tasks: dependencies.yaml
+  loop: "{{ chart['dependencies'] | default([]) }}"
+  loop_control:
+    loop_var: chart_dependency
+
+- name: create unique chart version
+  shell:
+    executable: /bin/bash
+    cmd: |
+      sha=$(sha256sum <<< "{{ chart | to_json }}" | cut -f1 -d' ')
+      version=$(helm show chart /tmp/{{ chart['name'] | quote }}/{{ chart['path'] | quote }} | yq -r .version)
+      printf "%s+source.%s" "$version" "$sha"
+  register: chart_version
+
+- name: package charts into /charts directory
+  shell:
+    cmd: >
+      helm package --destination=/charts
+      {{ '--dependency-update' if not chart.get('dependencies') }}
+      {{ '--version=' + chart.get('chart_version', chart_version.stdout) }}
+      /tmp/{{ chart['name'] }}/{{ chart['path'] }}

helm-chart-collator/playbooks/roles/install_helm_repo_charts/tasks/main.yaml

@@ -0,0 +1,8 @@
+---
+- name: setup repositories
+  command: helm repo add {{ chart["repo"] }} {{ chart["url"] }}
+
+- name: pull charts
+  command: helm pull {{ chart["repo"] }}/{{ chart["name"] }} \
+    --destination=/charts \
+    --version={{ chart["version"] }}

helm-chart-collator/playbooks/roles/install_tarred_charts/tasks/main.yaml

@@ -0,0 +1,5 @@
+---
+- name: download chart
+  get_url:
+    dest: /charts
+    url: "{{ chart['url'] }}"