Add image sync into the PR flow

This commit is contained in:
Matt Pryor 2023-01-18 18:18:39 +00:00
parent bd07004b57
commit 1bfc80a269
17 changed files with 439 additions and 223 deletions

View File

@ -1,19 +1,6 @@
name: helm lint
on:
pull_request:
branches:
- main
paths:
- .github/workflows/lint.yaml
- .github/workflows/install.yaml
- charts/**
push:
branches:
- main
paths:
- .github/workflows/lint.yaml
- .github/workflows/install.yaml
- charts/**
workflow_call:
jobs:
lint:
@ -38,7 +25,7 @@ jobs:
uses: scrungus/chart-testing-action@v3.7.3
- name: Dummy cloud creds
uses: "DamianReeves/write-file-action@master"
uses: DamianReeves/write-file-action@master
with:
path: clouds.yml
write-mode: overwrite
@ -55,13 +42,10 @@ jobs:
identity_api_version: 3
auth_type: "v3applicationcredential"
- name: Run chart-testing (lint)
run: ct lint --target-branch ${{ github.event.repository.default_branch }} --all --validate-maintainers=false --extra-values clouds.yml
install:
needs:
- lint
uses: ./.github/workflows/install.yaml
secrets: inherit
if: github.repository == 'stackhpc/capi-helm-charts'
run: |-
ct lint \
--target-branch ${{ github.event.repository.default_branch }} \
--all \
--validate-maintainers=false \
--extra-values clouds.yml

View File

@ -1,17 +1,6 @@
name: sync images
on:
pull_request:
branches:
- main
paths:
- .github/workflows/sync-images.yaml
- skopeo-manifests/**
push:
branches:
- main
paths:
- .github/workflows/sync-images.yaml
- skopeo-manifests/**
workflow_call:
jobs:
sync_images:
@ -21,17 +10,17 @@ jobs:
component:
- calico
- ccm-openstack
# - cilium
# - cluster-autoscaler
# - csi-cinder
# - ingress-nginx
# - kube-prometheus-stack
# - kubernetes-dashboard
# - loki-stack
# - mellanox-network-operator
# - metrics-server
# - nfd
# - nvidia-gpu-operator
- cilium
- cluster-autoscaler
- csi-cinder
- ingress-nginx
- kube-prometheus-stack
- kubernetes-dashboard
- loki-stack
- mellanox-network-operator
- metrics-server
- nfd
- nvidia-gpu-operator
steps:
- name: Check out the repository
uses: actions/checkout@v3

206
.github/workflows/includes/test.yaml vendored Normal file
View File

@ -0,0 +1,206 @@
name: test helm charts
on:
workflow_call:
jobs:
install:
runs-on: ubuntu-latest
strategy:
matrix:
version: [kube-1-23-9, kube-1-24-2, kube-1-25-4]
capi: ["v1.2.5"]
capo: ["v0.7.0-stackhpc.1"]
addonprovider: ["0.1.0-dev.0.main.21"]
continue-on-error: true # don't cancel all jobs if one fails - cleanup
steps:
- name: Checkout
uses: actions/checkout@v3
- name: Install tools
run: sudo apt install -y zip unzip
- name: Install sonobuoy
run: >
wget https://github.com/vmware-tanzu/sonobuoy/releases/download/v0.56.12/sonobuoy_0.56.12_linux_amd64.tar.gz
&& tar -xf sonobuoy_0.56.12_linux_amd64.tar.gz
&& sudo mv -n sonobuoy /usr/bin/
- uses: actions/setup-python@v4
with:
python-version: '3.9'
check-latest: true
- name: Set up Helm
uses: azure/setup-helm@v3
with:
version: v3.10.0
- name: Create k8s Kind Cluster
uses: helm/kind-action@v1.5.0
- name: Install cert-manager
run: |-
helm upgrade cert-manager cert-manager \
--repo https://charts.jetstack.io \
--version v1.10.1 \
--namespace cert-manager \
--create-namespace \
--install \
--set installCRDs=true \
--wait \
--timeout 5m
- name: Ensure Cluster API kustomization directory exists
run: mkdir -p clusterapi
- name: Write Cluster API kustomization file
uses: DamianReeves/write-file-action@master
with:
path: clusterapi/kustomization.yaml
write-mode: overwrite
contents: |
resources:
- https://github.com/kubernetes-sigs/cluster-api/releases/download/${{ matrix.capi }}/cluster-api-components.yaml
- https://github.com/stackhpc/cluster-api-provider-openstack/releases/download/${{ matrix.capo }}/infrastructure-components.yaml
patches:
- patch: |-
- op: replace
path: /spec/template/spec/containers/0/args
value:
- --leader-elect
- --metrics-bind-addr=localhost:8080
target:
kind: Deployment
namespace: capi-system
name: capi-controller-manager
- patch: |-
- op: replace
path: /spec/template/spec/containers/0/args
value:
- --leader-elect
- --metrics-bind-addr=localhost:8080
target:
kind: Deployment
namespace: capi-kubeadm-bootstrap-system
name: capi-kubeadm-bootstrap-controller-manager
- patch: |-
- op: replace
path: /spec/template/spec/containers/0/args
value:
- --leader-elect
- --metrics-bind-addr=localhost:8080
target:
kind: Deployment
namespace: capi-kubeadm-control-plane-system
name: capi-kubeadm-control-plane-controller-manager
- name: Install Cluster API resources
run: kubectl apply -k clusterapi/
- name: Wait for Cluster API controllers to become ready
run: |-
kubectl rollout status deployment/capi-controller-manager \
--namespace capi-system \
--timeout 5m \
&& \
kubectl rollout status deployment/capi-kubeadm-bootstrap-controller-manager \
--namespace capi-kubeadm-bootstrap-system \
--timeout 5m \
&& \
kubectl rollout status deployment/capi-kubeadm-control-plane-controller-manager \
--namespace capi-kubeadm-control-plane-system \
--timeout 5m \
&& \
kubectl rollout status deployment/capo-controller-manager \
--namespace capo-system \
--timeout 10m
- name: Install Cluster API add-on provider
run: |-
helm upgrade cluster-api-addon-provider cluster-api-addon-provider \
--repo https://stackhpc.github.io/cluster-api-addon-provider \
--version ${{ matrix.addonprovider }}
--namespace capi-addon-system \
--create-namespace \
--install \
--wait \
--timeout 5m
- name: Write cloud credential
run: 'echo "$TYLER_CLOUDS" > clouds.yml'
shell: bash
env:
TYLER_CLOUDS: ${{ secrets.TYLER_CLOUDS }}
- name: Create cluster
run: |-
helm upgrade ci-${GITHUB_SHA::7}-${{ matrix.version }} ./charts/openstack-cluster \
--install \
--dependency-update \
--values clouds.yml \
--values charts/openstack-cluster/ci/${{ matrix.version }}-values.yaml
# debug
#- name: Setup upterm session
# uses: lhotari/action-upterm@v1
- name: Wait for cluster to become ready
run: |-
kubectl wait clusters.cluster.x-k8s.io/ci-${GITHUB_SHA::7}-${{ matrix.version }} \
--for=condition=Ready \
--timeout 30m
- name: Wait for addons to deploy
run: |-
kubectl wait manifests \
--all \
--for=jsonpath='{.status.phase}'=Deployed \
--timeout 20m \
&& \
kubectl wait helmreleases \
--all \
--for=jsonpath='{.status.phase}'=Deployed \
--timeout 20m
- name: Write kubeconfig
run: |-
kubectl get secret ci-${GITHUB_SHA::7}-${{ matrix.version }}-kubeconfig \
-o go-template='{{ .data.value | base64decode }}' \
> kubeconfig
- name: Wait for dns pods
run: |-
kubectl wait po \
--namespace kube-system \
--selector k8s-app=kube-dns \
--for=condition=ready \
--timeout 20m
env:
KUBECONFIG: ./kubeconfig
- name: Run sonobuoy [smoke]
run: sonobuoy run --mode quick --wait
env:
KUBECONFIG: ./kubeconfig
#- name: Clean sonobuoy
# run: sonobuoy delete --kubeconfig kubeconfig-${{ matrix.version }} --wait --all
#- name: Upgrade cluster
# run: helm upgrade test-${GITHUB_SHA::7}-${{ matrix.version }} charts/openstack-cluster -n namespace-${GITHUB_SHA::7}-${{ matrix.version }} --install -f clouds.yml -f charts/openstack-cluster/ci/${{ matrix.version }}-values-upgrade.yaml --dependency-update
#- name: Wait for cluster READY
# run: kubectl wait -n namespace-${GITHUB_SHA::7}-${{ matrix.version }} clusters.cluster.x-k8s.io/test-${GITHUB_SHA::7}-${{ matrix.version }} --for=condition=Ready --timeout 20m
#- name: Wait for addons manifest
# run: kubectl wait manifests --all --for=jsonpath='{.status.phase}'=Deployed --namespace namespace-${GITHUB_SHA::7}-${{ matrix.version }} --selector capi.stackhpc.com/cluster=test-${GITHUB_SHA::7}-${{ matrix.version }} --timeout 20m
#- name: Wait for addons helmreleases
# run: kubectl wait helmreleases --all --for=jsonpath='{.status.phase}'=Deployed --namespace namespace-${GITHUB_SHA::7}-${{ matrix.version }} --selector capi.stackhpc.com/cluster=test-${GITHUB_SHA::7}-${{ matrix.version }} --timeout 20m
#- name: Run sonobuoy [smoke]
# run: sonobuoy run --mode quick --kubeconfig kubeconfig-${{ matrix.version }} --wait
- name: Delete release
if: ${{ always() }}
run: helm delete ci-${GITHUB_SHA::7}-${{ matrix.version }} --wait

View File

@ -1,158 +0,0 @@
name: Install Helm Charts
on: [workflow_call]
jobs:
install:
runs-on: ubuntu-latest
strategy:
matrix:
version: [kube-1-23-9, kube-1-24-2, kube-1-25-4]
capi: ["v1.2.5"]
capo: ["v0.7.0-stackhpc.1"]
addonprovider: ["0.1.0-dev.0.main.21"]
continue-on-error: true #don't cancel all jobs if one fails - cleanup
steps:
- name: Checkout
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Install tools
run: sudo apt install -y zip unzip
- name: Check if sonobuoy is installed
id: sonobuoy_exists
uses: andstor/file-existence-action@v2
with:
files: /usr/bin/sonobuoy
- name: Install sonobuoy
if: steps.sonobuoy_exists.outputs.files_exists == 'false'
run: >
wget https://github.com/vmware-tanzu/sonobuoy/releases/download/v0.56.12/sonobuoy_0.56.12_linux_amd64.tar.gz
&& tar -xf sonobuoy_0.56.12_linux_amd64.tar.gz
&& sudo mv -n sonobuoy /usr/bin/
- uses: actions/setup-python@v4
with:
python-version: '3.9'
check-latest: true
- name: Set up Helm
uses: azure/setup-helm@v3
with:
version: v3.10.0
- name: Copy cloud creds to file
run: 'echo "$TYLER_CLOUDS" > clouds.yml'
shell: bash
env:
TYLER_CLOUDS: ${{ secrets.TYLER_CLOUDS }}
- name: Create k8s Kind Cluster
uses: helm/kind-action@v1.5.0
- name: Install cert-manager
run: >
helm repo add jetstack https://charts.jetstack.io
&& helm repo update
&& helm install cert-manager jetstack/cert-manager --namespace cert-manager --create-namespace --version v1.10.1 --set installCRDs=true --wait
- name: Make kustomization directory
run: mkdir clusterapi
# From here: https://github.com/stackhpc/ansible-collection-azimuth-ops/blob/main/roles/clusterapi/defaults/main.yml
- name: Overwrite file
uses: "DamianReeves/write-file-action@master"
with:
path: clusterapi/kustomization.yaml
write-mode: overwrite
contents: |
patches:
- patch: "- op: replace\n path: /spec/template/spec/containers/0/args\n value:\n
\ - --leader-elect\n - --metrics-bind-addr=localhost:8080"
target:
kind: Deployment
name: capi-controller-manager
namespace: capi-system
- patch: "- op: replace\n path: /spec/template/spec/containers/0/args\n value:\n
\ - --leader-elect\n - --metrics-bind-addr=localhost:8080"
target:
kind: Deployment
name: capi-kubeadm-bootstrap-controller-manager
namespace: capi-kubeadm-bootstrap-system
- patch: "- op: replace\n path: /spec/template/spec/containers/0/args\n value:\n
\ - --leader-elect\n - --metrics-bind-addr=localhost:8080"
target:
kind: Deployment
name: capi-kubeadm-control-plane-controller-manager
namespace: capi-kubeadm-control-plane-system
resources:
- https://github.com/kubernetes-sigs/cluster-api/releases/download/${{ matrix.capi }}/cluster-api-components.yaml
- https://github.com/stackhpc/cluster-api-provider-openstack/releases/download/${{ matrix.capo }}/infrastructure-components.yaml
- name: Install Cluster API resources
run: kubectl apply -k clusterapi/
- name: Wait for Cluster API controllers to become ready
run: >
kubectl rollout status --namespace capi-system deployment/capi-controller-manager --timeout 10m
&& kubectl rollout status --namespace capi-kubeadm-bootstrap-system deployment/capi-kubeadm-bootstrap-controller-manager --timeout 10m
&& kubectl rollout status --namespace capi-kubeadm-control-plane-system deployment/capi-kubeadm-control-plane-controller-manager --timeout 10m
&& kubectl rollout status --namespace capo-system deployment/capo-controller-manager --timeout 10m
- name: Install Cluster API add-on provider
run: >
helm install --repo https://stackhpc.github.io/cluster-api-addon-provider cluster-api-addon-provider --version ${{ matrix.addonprovider }} -n capi-addon-system --create-namespace --wait --timeout 30m cluster-api-addon-provider
- name: Install chart
run: helm upgrade test-${GITHUB_SHA::7}-${{ matrix.version }} charts/openstack-cluster -n namespace-${GITHUB_SHA::7}-${{ matrix.version }} --create-namespace --install -f clouds.yml -f charts/openstack-cluster/ci/${{ matrix.version }}-values.yaml --dependency-update
# debug
#- name: Setup upterm session
# uses: lhotari/action-upterm@v1
- name: Wait for cluster READY
run: kubectl wait -n namespace-${GITHUB_SHA::7}-${{ matrix.version }} clusters.cluster.x-k8s.io/test-${GITHUB_SHA::7}-${{ matrix.version }} --for=condition=Ready --timeout 30m
- name: Wait for addons manifest
run: kubectl wait manifests --all --for=jsonpath='{.status.phase}'=Deployed --namespace namespace-${GITHUB_SHA::7}-${{ matrix.version }} --selector capi.stackhpc.com/cluster=test-${GITHUB_SHA::7}-${{ matrix.version }} --timeout 20m
- name: Wait for addons helmreleases
run: kubectl wait helmreleases --all --for=jsonpath='{.status.phase}'=Deployed --namespace namespace-${GITHUB_SHA::7}-${{ matrix.version }} --selector capi.stackhpc.com/cluster=test-${GITHUB_SHA::7}-${{ matrix.version }} --timeout 20m
- name: Grab kubeconfig
run: kubectl -n namespace-${GITHUB_SHA::7}-${{ matrix.version }} get secret test-${GITHUB_SHA::7}-${{ matrix.version }}-kubeconfig -o jsonpath='{.data.value}' | base64 --decode > kubeconfig-${{ matrix.version }}
- name: Wait for dns pods
run: kubectl --kubeconfig kubeconfig-${{ matrix.version }} wait po -n kube-system -l k8s-app=kube-dns --for=condition=ready --timeout 20m
- name: Run sonobuoy [smoke]
run: sonobuoy run --mode quick --kubeconfig kubeconfig-${{ matrix.version }} --wait
#- name: Clean sonobuoy
# run: sonobuoy delete --kubeconfig kubeconfig-${{ matrix.version }} --wait --all
#- name: Upgrade cluster
# run: helm upgrade test-${GITHUB_SHA::7}-${{ matrix.version }} charts/openstack-cluster -n namespace-${GITHUB_SHA::7}-${{ matrix.version }} --install -f clouds.yml -f charts/openstack-cluster/ci/${{ matrix.version }}-values-upgrade.yaml --dependency-update
#- name: Wait for cluster READY
# run: kubectl wait -n namespace-${GITHUB_SHA::7}-${{ matrix.version }} clusters.cluster.x-k8s.io/test-${GITHUB_SHA::7}-${{ matrix.version }} --for=condition=Ready --timeout 20m
#- name: Wait for addons manifest
# run: kubectl wait manifests --all --for=jsonpath='{.status.phase}'=Deployed --namespace namespace-${GITHUB_SHA::7}-${{ matrix.version }} --selector capi.stackhpc.com/cluster=test-${GITHUB_SHA::7}-${{ matrix.version }} --timeout 20m
#- name: Wait for addons helmreleases
# run: kubectl wait helmreleases --all --for=jsonpath='{.status.phase}'=Deployed --namespace namespace-${GITHUB_SHA::7}-${{ matrix.version }} --selector capi.stackhpc.com/cluster=test-${GITHUB_SHA::7}-${{ matrix.version }} --timeout 20m
#- name: Run sonobuoy [smoke]
# run: sonobuoy run --mode quick --kubeconfig kubeconfig-${{ matrix.version }} --wait
- name: Delete release
if: ${{ always() }}
run: helm delete -n namespace-${GITHUB_SHA::7}-${{ matrix.version }} test-${GITHUB_SHA::7}-${{ matrix.version }} --wait
- name: Delete namespace
if: ${{ always() }}
run: kubectl delete namespace namespace-${GITHUB_SHA::7}-${{ matrix.version }}

23
.github/workflows/pr.yaml vendored Normal file
View File

@ -0,0 +1,23 @@
name: test pr
on:
pull_request:
branches:
- main
jobs:
# Run the lint on every PR, even from external repos
lint:
uses: .github/workflows/includes/lint.yaml
# Only run image sync and test on the main repo
sync_images:
needs: [lint]
uses: .github/workflows/includes/sync-images.yaml
secrets: inherit
if: github.repository == 'stackhpc/capi-helm-charts'
test:
needs: [sync_images]
uses: .github/workflows/includes/test.yaml
secrets: inherit
if: github.repository == 'stackhpc/capi-helm-charts'

View File

@ -1,21 +1,21 @@
# docker.io:
# images:
# calico/apiserver:
# - v3.24.5
# calico/cni:
# - v3.24.5
# calico/csi:
# - v3.24.5
# calico/kube-controllers:
# - v3.24.5
# calico/node:
# - v3.24.5
# calico/node-driver-registrar:
# - v3.24.5
# calico/pod2daemon-flexvol:
# - v3.24.5
# calico/typha:
# - v3.24.5
docker.io:
images:
calico/apiserver:
- v3.24.5
calico/cni:
- v3.24.5
calico/csi:
- v3.24.5
calico/kube-controllers:
- v3.24.5
calico/node:
- v3.24.5
calico/node-driver-registrar:
- v3.24.5
calico/pod2daemon-flexvol:
- v3.24.5
calico/typha:
- v3.24.5
quay.io:
images:

View File

@ -0,0 +1,18 @@
quay.io:
images:
cilium/certgen:
- v0.1.8
cilium/cilium:
- v1.12.5
cilium/clustermesh-apiserver:
- v1.12.5
cilium/hubble-relay:
- v1.12.5
cilium/hubble-ui-backend:
- v0.9.2
cilium/hubble-ui:
- v0.9.2
cilium/operator:
- v1.12.5
cilium/startup-script:
- d69851597ea019af980891a4628fb36b7880ec26

View File

@ -0,0 +1,4 @@
k8s.gcr.io:
images:
autoscaling/cluster-autoscaler:
- v1.26.1

View File

@ -0,0 +1,19 @@
docker.io:
images:
k8scloudprovider/cinder-csi-plugin:
- v1.25.0
k8s.gcr.io:
images:
sig-storage/csi-attacher:
- v3.4.0
sig-storage/csi-node-driver-registrar:
- v2.5.0
sig-storage/csi-provisioner:
- v3.1.0
sig-storage/csi-resizer:
- v1.4.0
sig-storage/csi-snapshotter:
- v5.0.1
sig-storage/livenessprobe:
- v2.6.0

View File

@ -0,0 +1,8 @@
registry.k8s.io:
images:
defaultbackend-amd64:
- 1.5
ingress-nginx/controller:
- v1.5.1
ingress-nginx/kube-webhook-certgen:
- v20220916-gd32f8c343

View File

@ -0,0 +1,28 @@
docker.io:
images:
grafana/grafana:
- 9.3.1
quay.io:
images:
kiwigrid/k8s-sidecar:
- 1.21.0
prometheus/alertmanager:
- v0.25.0
prometheus/node-exporter:
- v1.5.0
prometheus/prometheus:
- v2.40.5
prometheus-operator/prometheus-config-reloader:
- v0.61.1
prometheus-operator/prometheus-operator:
- v0.61.1
thanos/thanos:
- v0.29.0
registry.k8s.io:
images:
ingress-nginx/kube-webhook-certgen:
- v1.3.0
kube-state-metrics/kube-state-metrics:
- v2.7.0

View File

@ -0,0 +1,6 @@
docker.io:
images:
kubernetesui/dashboard:
- v2.7.0
kubernetesui/metrics-scraper:
- v1.0.8

View File

@ -0,0 +1,6 @@
docker.io:
images:
grafana/loki:
- 2.6.1
grafana/promtail:
- 2.7.0

View File

@ -0,0 +1,36 @@
#####
# This file defines images that are required for deployments of the CAPI Helm charts,
# allowing them to be synchronised to another repository using skopeo
# In particular, they are synchronised to ghcr.io/stackhpc/capi, which is used in the
# CAPI Helm charts by default
#####
docker.io:
images:
mellanox/nv-peer-mem-driver-1.1-0:
- amd64-ubuntu20.04
ghcr.io:
images:
k8snetworkplumbingwg/multus-cni:
- v3.8
k8snetworkplumbingwg/plugins:
- v0.8.7-amd64
k8snetworkplumbingwg/sriov-network-device-plugin:
- v3.5.1
k8snetworkplumbingwg/whereabouts:
- v0.5.2-amd64
mellanox/ib-kubernetes:
- v1.0.2
nvcr.io:
images:
nvidia/cloud-native/ipoib-cni:
- v1.1.0
nvidia/cloud-native/k8s-rdma-shared-dev-plugin:
- v1.3.2
nvidia/cloud-native/network-operator:
- v1.4.0
nvidia/mellanox/mofed:
- 5.8-1.0.1.1.2-ubuntu20.04-amd64

View File

@ -0,0 +1,4 @@
k8s.gcr.io:
images:
metrics-server/metrics-server:
- v0.6.2

4
skopeo-manifests/nfd.yml Normal file
View File

@ -0,0 +1,4 @@
registry.k8s.io:
images:
nfd/node-feature-discovery:
- v0.12.0

View File

@ -0,0 +1,39 @@
#####
# This file defines images that are required for deployments of the CAPI Helm charts,
# allowing them to be synchronised to another repository using skopeo
# In particular, they are synchronised to ghcr.io/stackhpc/capi, which is used in the
# CAPI Helm charts by default
#####
nvcr.io:
images:
nvidia/cloud-native/dcgm:
- 3.1.3-1-ubuntu20.04
nvidia/cloud-native/gpu-operator-validator:
- v22.9.1
nvidia/cloud-native/k8s-driver-manager:
- v0.5.1
nvidia/cloud-native/k8s-mig-manager:
- v0.5.0-ubuntu20.04
nvidia/cloud-native/nvidia-fs:
- 2.14.13-ubuntu20.04
nvidia/cloud-native/vgpu-device-manager:
- v0.2.0
nvidia/cuda:
- 11.7.1-base-ubi8
- 11.8.0-base-ubi8
nvidia/driver:
- 525.60.13-ubuntu20.04
nvidia/gpu-feature-discovery:
- v0.7.0-ubi8
nvidia/gpu-operator:
- v22.9.1
nvidia/kubevirt-gpu-device-plugin:
- v1.2.1
nvidia/k8s/container-toolkit:
- v1.11.0-ubuntu20.04
nvidia/k8s/dcgm-exporter:
- 3.1.3-3.1.2-ubuntu20.04
nvidia/k8s-device-plugin:
- v0.13.0-ubi8