Test a matrix of k8s versions (#24)

Run for a matrix of k8s versions this test:

* Create an ephemeral management cluster with Kind
* Create cluster with specified values
* Run sonobuoy smoke tests
* TODO: upgrade to next version
* Tear down cluster

Co-authored-by: John Garbutt <john.garbutt@stackhpc.com>

.github/workflows/install.yaml

@@ -3,12 +3,32 @@ on: [workflow_call]
 
 jobs:
   install:
-    runs-on: self-hosted
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        version: [kube-1-23-9, kube-1-24-2, kube-1-25-4]
+    continue-on-error: true #don't cancel all jobs if one fails - cleanup
     steps:
       - name: Checkout
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
+      
+      - name: Install tools
+        run: sudo apt install -y zip unzip 
+      
+      - name: Check if sonobuoy is installed
+        id: sonobuoy_exists
+        uses: andstor/file-existence-action@v2
+        with:
+          files: /usr/bin/sonobuoy
+      
+      - name: Install sonobuoy
+        if: steps.sonobuoy_exists.outputs.files_exists == 'false'
+        run: >
+          wget https://github.com/vmware-tanzu/sonobuoy/releases/download/v0.56.12/sonobuoy_0.56.12_linux_amd64.tar.gz
+          && tar -xf sonobuoy_0.56.12_linux_amd64.tar.gz
+          && sudo mv -n sonobuoy /usr/bin/
 
       - uses: actions/setup-python@v4
         with:
@@ -19,9 +39,6 @@ jobs:
         uses: azure/setup-helm@v3
         with:
           version: v3.10.0
-          
-      - name: Set up chart-testing
-        uses: scrungus/chart-testing-action@v3.7.4
       
       - name: Copy cloud creds to file
         run: 'echo "$TYLER_CLOUDS" > clouds.yml'
@@ -29,5 +46,110 @@ jobs:
         env:
           TYLER_CLOUDS: ${{ secrets.TYLER_CLOUDS }}
       
-      - name: Run Helm Install
-        run: ct install --target-branch ${{ github.event.repository.default_branch }} --all --extra-values clouds.yml --helm-extra-args "--timeout 5m" --openstack-deploy
+      - name: Create k8s Kind Cluster
+        uses: helm/kind-action@v1.5.0
+      
+      - name: Install cert-manager
+        run: >
+          helm repo add jetstack https://charts.jetstack.io
+          && helm repo update 
+          && helm install cert-manager jetstack/cert-manager --namespace cert-manager --create-namespace --version v1.10.1 --set installCRDs=true --wait
+      
+      - name: Make kustomization directory 
+        run: mkdir clusterapi
+      
+      # From here: https://github.com/stackhpc/ansible-collection-azimuth-ops/blob/main/roles/clusterapi/defaults/main.yml
+      - name: Overwrite file
+        uses: "DamianReeves/write-file-action@master"
+        with:
+          path: clusterapi/kustomization.yaml
+          write-mode: overwrite
+          contents: |
+            patches:
+            -   patch: "- op: replace\n  path: /spec/template/spec/containers/0/args\n  value:\n
+                    \   - --leader-elect\n    - --metrics-bind-addr=localhost:8080"
+                target:
+                    kind: Deployment
+                    name: capi-controller-manager
+                    namespace: capi-system
+            -   patch: "- op: replace\n  path: /spec/template/spec/containers/0/args\n  value:\n
+                    \   - --leader-elect\n    - --metrics-bind-addr=localhost:8080"
+                target:
+                    kind: Deployment
+                    name: capi-kubeadm-bootstrap-controller-manager
+                    namespace: capi-kubeadm-bootstrap-system
+            -   patch: "- op: replace\n  path: /spec/template/spec/containers/0/args\n  value:\n
+                    \   - --leader-elect\n    - --metrics-bind-addr=localhost:8080"
+                target:
+                    kind: Deployment
+                    name: capi-kubeadm-control-plane-controller-manager
+                    namespace: capi-kubeadm-control-plane-system
+            resources:
+            - https://github.com/kubernetes-sigs/cluster-api/releases/download/v1.2.5/cluster-api-components.yaml
+            - https://github.com/stackhpc/cluster-api-provider-openstack/releases/download/v0.7.0-stackhpc.1/infrastructure-components.yaml
+      
+      - name: Install Cluster API resources
+        run: kubectl apply -k clusterapi/
+
+      - name: Wait for Cluster API controllers to become ready
+        run: >
+          kubectl rollout status --namespace capi-system deployment/capi-controller-manager --timeout 10m
+          && kubectl rollout status --namespace capi-kubeadm-bootstrap-system deployment/capi-kubeadm-bootstrap-controller-manager --timeout 10m
+          && kubectl rollout status --namespace capi-kubeadm-control-plane-system deployment/capi-kubeadm-control-plane-controller-manager --timeout 10m
+          && kubectl rollout status --namespace capo-system deployment/capo-controller-manager --timeout 10m
+      
+      
+      - name: Install Cluster API add-on provider
+        run: >
+          helm install --repo https://stackhpc.github.io/cluster-api-addon-provider cluster-api-addon-provider --version 0.1.0-dev.0.main.21 -n capi-addon-system --create-namespace --wait --timeout 30m cluster-api-addon-provider
+
+      - name: Install chart 
+        run: helm upgrade test-${GITHUB_SHA::7}-${{ matrix.version }} charts/openstack-cluster -n namespace-${GITHUB_SHA::7}-${{ matrix.version }} --create-namespace --install -f clouds.yml -f charts/openstack-cluster/ci/${{ matrix.version }}-values.yaml --dependency-update
+
+      # debug
+      #- name: Setup upterm session
+      #  uses: lhotari/action-upterm@v1
+
+      - name: Wait for cluster READY
+        run: kubectl wait -n namespace-${GITHUB_SHA::7}-${{ matrix.version }} clusters.cluster.x-k8s.io/test-${GITHUB_SHA::7}-${{ matrix.version }} --for=condition=Ready --timeout 30m
+
+      - name: Wait for addons manifest
+        run: kubectl wait manifests --all --for=jsonpath='{.status.phase}'=Deployed --namespace namespace-${GITHUB_SHA::7}-${{ matrix.version }} --selector capi.stackhpc.com/cluster=test-${GITHUB_SHA::7}-${{ matrix.version }} --timeout 20m
+      
+      - name: Wait for addons helmreleases 
+        run: kubectl wait helmreleases --all --for=jsonpath='{.status.phase}'=Deployed --namespace namespace-${GITHUB_SHA::7}-${{ matrix.version }} --selector capi.stackhpc.com/cluster=test-${GITHUB_SHA::7}-${{ matrix.version }} --timeout 20m
+
+      - name: Grab kubeconfig 
+        run: kubectl -n namespace-${GITHUB_SHA::7}-${{ matrix.version }} get secret test-${GITHUB_SHA::7}-${{ matrix.version }}-kubeconfig -o jsonpath='{.data.value}' | base64 --decode > kubeconfig-${{ matrix.version }}
+
+      - name: Wait for dns pods 
+        run: kubectl --kubeconfig kubeconfig-${{ matrix.version }} wait po -n kube-system -l k8s-app=kube-dns --for=condition=ready --timeout 20m
+
+      - name: Run sonobuoy [smoke]
+        run: sonobuoy run --mode quick --kubeconfig kubeconfig-${{ matrix.version }} --wait
+
+      #- name: Clean sonobuoy 
+      #  run: sonobuoy delete --kubeconfig kubeconfig-${{ matrix.version }} --wait --all
+      
+      #- name: Upgrade cluster
+      #  run: helm upgrade test-${GITHUB_SHA::7}-${{ matrix.version }} charts/openstack-cluster -n namespace-${GITHUB_SHA::7}-${{ matrix.version }} --install -f clouds.yml -f charts/openstack-cluster/ci/${{ matrix.version }}-values-upgrade.yaml --dependency-update
+      
+      #- name: Wait for cluster READY
+      #  run: kubectl wait -n namespace-${GITHUB_SHA::7}-${{ matrix.version }} clusters.cluster.x-k8s.io/test-${GITHUB_SHA::7}-${{ matrix.version }} --for=condition=Ready --timeout 20m
+
+      #- name: Wait for addons manifest
+      #  run: kubectl wait manifests --all --for=jsonpath='{.status.phase}'=Deployed --namespace namespace-${GITHUB_SHA::7}-${{ matrix.version }} --selector capi.stackhpc.com/cluster=test-${GITHUB_SHA::7}-${{ matrix.version }} --timeout 20m
+      
+      #- name: Wait for addons helmreleases 
+      #  run: kubectl wait helmreleases --all --for=jsonpath='{.status.phase}'=Deployed --namespace namespace-${GITHUB_SHA::7}-${{ matrix.version }} --selector capi.stackhpc.com/cluster=test-${GITHUB_SHA::7}-${{ matrix.version }} --timeout 20m
+
+      #- name: Run sonobuoy [smoke]
+      #  run: sonobuoy run --mode quick --kubeconfig kubeconfig-${{ matrix.version }} --wait
+        
+      - name: Delete release
+        if: ${{ always() }}
+        run: helm delete -n namespace-${GITHUB_SHA::7}-${{ matrix.version }} test-${GITHUB_SHA::7}-${{ matrix.version }} --wait
+      
+      - name: Delete namespace 
+        if: ${{ always() }}
+        run: kubectl delete namespace namespace-${GITHUB_SHA::7}-${{ matrix.version }}

charts/openstack-cluster/ci/kube-1-23-9-ha-values.yaml

@@ -0,0 +1,12 @@
+---
+kubernetesVersion: 1.23.9
+
+machineImageId: e8c65726-2667-4d22-a45e-80eb19311a8d
+
+controlPlane:
+  machineFlavor: vm.alaska.cpu.general.small
+
+nodeGroups:
+  - machineCount: 2
+    machineFlavor: vm.alaska.cpu.general.small
+    name: test-group1

charts/openstack-cluster/ci/kube-1-23-9-values-upgrade.yaml

@@ -0,0 +1 @@
+kube-1-24-2-values.yaml

charts/openstack-cluster/ci/kube-1-23-9-values.yaml

@@ -0,0 +1,16 @@
+---
+kubernetesVersion: 1.23.9
+
+machineImageId: e8c65726-2667-4d22-a45e-80eb19311a8d
+
+apiServer:
+  enableLoadBalancer: false
+
+controlPlane:
+  machineFlavor: vm.alaska.cpu.general.small
+  machineCount: 1
+
+nodeGroups:
+  - machineCount: 2
+    machineFlavor: vm.alaska.cpu.general.small
+    name: test-group1

charts/openstack-cluster/ci/kube-1-24-2-values-upgrade.yaml

@@ -0,0 +1 @@
+kube-1-25-4-values.yaml

charts/openstack-cluster/ci/kube-1-24-2-values.yaml

@@ -0,0 +1,16 @@
+---
+kubernetesVersion: 1.24.2
+
+machineImageId: 9b5d513c-67b4-4fd6-8bf1-3bf4525c04c3
+
+apiServer:
+  enableLoadBalancer: false
+
+controlPlane:
+  machineFlavor: vm.alaska.cpu.general.small
+  machineCount: 1
+
+nodeGroups:
+  - machineCount: 2
+    machineFlavor: vm.alaska.cpu.general.small
+    name: test-group1

charts/openstack-cluster/ci/kube-1-25-4-ha-values.yaml

@@ -0,0 +1,12 @@
+---
+kubernetesVersion: 1.25.4
+
+machineImageId: 48c078a5-fd89-4f61-9d6a-c4f48745c0ae
+
+controlPlane:
+  machineFlavor: vm.alaska.cpu.general.small
+
+nodeGroups:
+  - machineCount: 2
+    machineFlavor: vm.alaska.cpu.general.small
+    name: test-group1

charts/openstack-cluster/ci/kube-1-25-4-values-upgrade.yaml

@@ -0,0 +1 @@
+kube-1-25-4-values.yaml

charts/openstack-cluster/ci/kube-1-25-4-values.yaml

@@ -0,0 +1,16 @@
+---
+kubernetesVersion: 1.25.4
+
+machineImageId: 48c078a5-fd89-4f61-9d6a-c4f48745c0ae
+
+apiServer:
+  enableLoadBalancer: false
+
+controlPlane:
+  machineFlavor: vm.alaska.cpu.general.small
+  machineCount: 1
+
+nodeGroups:
+  - machineCount: 2
+    machineFlavor: vm.alaska.cpu.general.small
+    name: test-group1