From 4be9dfda689fecd706b0552d169771e8038df84a Mon Sep 17 00:00:00 2001 From: Tin Lam Date: Thu, 10 Dec 2020 11:28:50 -0600 Subject: [PATCH] chore(tests): refactors test script This patch set refactors and cleans up the test for the cicd stack, and updates deprecated apiVersion. Signed-off-by: Tin Lam Change-Id: Ice3e678a677ecd4bc39e1fb840d49b3c93453452 --- .../mutatingwebhookconfiguration-webhook.yaml | 2 +- ...ngwebhookconfiguration-webhook-config.yaml | 2 +- ...bhookconfiguration-webhook-validation.yaml | 2 +- .../templates/config_validation-triggers.yaml | 2 +- .../templates/webhook-triggers.yaml | 2 +- .../webhook_validation-triggers.yaml | 2 +- tools/gate/harbor/300-test.sh | 5 + tools/gate/pipelines/run-test.sh | 26 ++++ tools/gate/tekton/300-test.sh | 49 ++++--- tools/utils/aio-setup.sh | 121 ++++++++++++++++++ tools/utils/harbor-tekton-aio.sh | 35 ----- tools/utils/tekton-cleanup.sh | 23 ++++ zuul.d/jobs.yaml | 6 +- 13 files changed, 206 insertions(+), 71 deletions(-) create mode 100755 tools/gate/harbor/300-test.sh create mode 100755 tools/gate/pipelines/run-test.sh create mode 100755 tools/utils/aio-setup.sh delete mode 100755 tools/utils/harbor-tekton-aio.sh create mode 100755 tools/utils/tekton-cleanup.sh diff --git a/charts/tekton-pipelines/templates/mutatingwebhookconfiguration-webhook.yaml b/charts/tekton-pipelines/templates/mutatingwebhookconfiguration-webhook.yaml index b53a53e4..a3fc029b 100644 --- a/charts/tekton-pipelines/templates/mutatingwebhookconfiguration-webhook.yaml +++ b/charts/tekton-pipelines/templates/mutatingwebhookconfiguration-webhook.yaml @@ -1,6 +1,6 @@ {{- define "mutatingwebhookconfiguration-webhook" -}} --- -apiVersion: admissionregistration.k8s.io/v1beta1 +apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration metadata: name: webhook.pipeline.tekton.dev diff --git a/charts/tekton-pipelines/templates/validatingwebhookconfiguration-webhook-config.yaml b/charts/tekton-pipelines/templates/validatingwebhookconfiguration-webhook-config.yaml index 46e4d6b4..ed6f2ee3 100644 --- a/charts/tekton-pipelines/templates/validatingwebhookconfiguration-webhook-config.yaml +++ b/charts/tekton-pipelines/templates/validatingwebhookconfiguration-webhook-config.yaml @@ -1,6 +1,6 @@ {{- define "validatingwebhookconfiguration-webhook-config" -}} --- -apiVersion: admissionregistration.k8s.io/v1beta1 +apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: name: config.webhook.pipeline.tekton.dev diff --git a/charts/tekton-pipelines/templates/validatingwebhookconfiguration-webhook-validation.yaml b/charts/tekton-pipelines/templates/validatingwebhookconfiguration-webhook-validation.yaml index 93a2813c..072bf585 100644 --- a/charts/tekton-pipelines/templates/validatingwebhookconfiguration-webhook-validation.yaml +++ b/charts/tekton-pipelines/templates/validatingwebhookconfiguration-webhook-validation.yaml @@ -1,6 +1,6 @@ {{- define "validatingwebhookconfiguration-webhook-validation" -}} --- -apiVersion: admissionregistration.k8s.io/v1beta1 +apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: name: validation.webhook.pipeline.tekton.dev diff --git a/charts/tekton-triggers/templates/config_validation-triggers.yaml b/charts/tekton-triggers/templates/config_validation-triggers.yaml index 922c021f..23640bcb 100644 --- a/charts/tekton-triggers/templates/config_validation-triggers.yaml +++ b/charts/tekton-triggers/templates/config_validation-triggers.yaml @@ -1,6 +1,6 @@ {{- define "config_validation-triggers" -}} --- -apiVersion: admissionregistration.k8s.io/v1beta1 +apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: name: config.webhook.triggers.tekton.dev diff --git a/charts/tekton-triggers/templates/webhook-triggers.yaml b/charts/tekton-triggers/templates/webhook-triggers.yaml index 7949e029..3f33b91a 100644 --- a/charts/tekton-triggers/templates/webhook-triggers.yaml +++ b/charts/tekton-triggers/templates/webhook-triggers.yaml @@ -1,6 +1,6 @@ {{- define "webhook-triggers" -}} --- -apiVersion: admissionregistration.k8s.io/v1beta1 +apiVersion: admissionregistration.k8s.io/v1 kind: MutatingWebhookConfiguration metadata: name: webhook.triggers.tekton.dev diff --git a/charts/tekton-triggers/templates/webhook_validation-triggers.yaml b/charts/tekton-triggers/templates/webhook_validation-triggers.yaml index 983502fc..252db19c 100644 --- a/charts/tekton-triggers/templates/webhook_validation-triggers.yaml +++ b/charts/tekton-triggers/templates/webhook_validation-triggers.yaml @@ -1,6 +1,6 @@ {{- define "webhook_validation-triggers" -}} --- -apiVersion: admissionregistration.k8s.io/v1beta1 +apiVersion: admissionregistration.k8s.io/v1 kind: ValidatingWebhookConfiguration metadata: name: validation.webhook.triggers.tekton.dev diff --git a/tools/gate/harbor/300-test.sh b/tools/gate/harbor/300-test.sh new file mode 100755 index 00000000..788a9422 --- /dev/null +++ b/tools/gate/harbor/300-test.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +set -eux + +helm test harbor -n harbor diff --git a/tools/gate/pipelines/run-test.sh b/tools/gate/pipelines/run-test.sh new file mode 100755 index 00000000..64695ac7 --- /dev/null +++ b/tools/gate/pipelines/run-test.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -eux + +# executes the harbor tests +./tools/gate/harbor/300-test.sh + +# executes the tekton tests +./tools/gate/tekton/300-test.sh + +# performs AIO integration tests +CREDENTIAL="--username=admin --password=Harbor12345" + +# Downloads the cli helm push plugin with chartmuseum +helm plugin install https://github.com/chartmuseum/helm-push + +# Downloads a chartmuseum tarball and upload it to a newly created repo in harbor +helm repo add stable https://charts.helm.sh/stable +helm repo update +helm pull stable/chartmuseum + +HARBORIP=$(kubectl -n harbor get svc harbor-harbor-core -ojsonpath='{.spec.clusterIP}') +helm repo add myrepo http://$HARBORIP/chartrepo $CREDENTIAL +helm push $CREDENTIAL $(ls chartmuseum*.tgz) myrepo +helm repo update +helm search repo --regexp myrepo/* diff --git a/tools/gate/tekton/300-test.sh b/tools/gate/tekton/300-test.sh index e4be9adb..6e2ba712 100755 --- a/tools/gate/tekton/300-test.sh +++ b/tools/gate/tekton/300-test.sh @@ -2,11 +2,12 @@ set -eux -NS="tekton-pipelines" +TEKTON_NS="tekton-pipelines" +# Runs the tekton pipeline trigger test function retry { local n=1 - local max=3 + local max=5 local delay=10 while true; do @@ -22,44 +23,40 @@ function retry { done } +sleep 60 -kubectl -n $NS apply -f ./tools/gate/tekton/yaml/role-resources/secret.yaml -kubectl -n $NS apply -f ./tools/gate/tekton/yaml/role-resources/serviceaccount.yaml -kubectl -n $NS apply -f ./tools/gate/tekton/yaml/role-resources/clustertriggerbinding-roles -kubectl -n $NS apply -f ./tools/gate/tekton/yaml/role-resources/triggerbinding-roles -retry kubectl -n $NS apply -f ./tools/gate/tekton/yaml/triggertemplates/triggertemplate.yaml -retry kubectl -n $NS apply -f ./tools/gate/tekton/yaml/triggerbindings/triggerbinding.yaml -retry kubectl -n $NS apply -f ./tools/gate/tekton/yaml/triggerbindings/triggerbinding-message.yaml -retry kubectl -n $NS apply -f ./tools/gate/tekton/yaml/eventlisteners/eventlistener.yaml +kubectl -n $TEKTON_NS apply -f ./tools/gate/tekton/yaml/role-resources/secret.yaml +kubectl -n $TEKTON_NS apply -f ./tools/gate/tekton/yaml/role-resources/serviceaccount.yaml +kubectl -n $TEKTON_NS apply -f ./tools/gate/tekton/yaml/role-resources/clustertriggerbinding-roles +kubectl -n $TEKTON_NS apply -f ./tools/gate/tekton/yaml/role-resources/triggerbinding-roles +retry kubectl -n $TEKTON_NS apply -f ./tools/gate/tekton/yaml/triggertemplates/triggertemplate.yaml +retry kubectl -n $TEKTON_NS apply -f ./tools/gate/tekton/yaml/triggerbindings/triggerbinding.yaml +retry kubectl -n $TEKTON_NS apply -f ./tools/gate/tekton/yaml/triggerbindings/triggerbinding-message.yaml +retry kubectl -n $TEKTON_NS apply -f ./tools/gate/tekton/yaml/eventlisteners/eventlistener.yaml -kubectl -n $NS get svc -kubectl -n $NS get pod -kubectl -n $NS get triggerbinding -kubectl -n $NS get triggertemplate +kubectl -n $TEKTON_NS get svc +kubectl -n $TEKTON_NS get pod +kubectl -n $TEKTON_NS get triggerbinding +kubectl -n $TEKTON_NS get triggertemplate -kubectl -n $NS wait --for=condition=Ready pod --timeout=120s --all +kubectl -n $TEKTON_NS wait --for=condition=Ready pod --timeout=120s --all # Install the pipeline -kubectl -n $NS apply -f ./tools/gate/tekton/yaml/example-pipeline.yaml -kubectl -n $NS wait --for=condition=Ready pod --timeout=120s --all +kubectl -n $TEKTON_NS apply -f ./tools/gate/tekton/yaml/example-pipeline.yaml +kubectl -n $TEKTON_NS wait --for=condition=Ready pod --timeout=120s --all kubectl get po -A # Trigger the sample github pipeline -SVCIP=$(kubectl -n $NS get svc --no-headers | grep el-listener | awk '{print $3}') +SVCIP=$(kubectl -n $TEKTON_NS get svc --no-headers | grep el-listener | awk '{print $3}') curl -X POST \ http://$SVCIP:8080 \ -H 'Content-Type: application/json' \ -H 'X-Hub-Signature: sha1=2da37dcb9404ff17b714ee7a505c384758ddeb7b' \ - -d '{ - "repository": - { - "url": "https://github.com/tektoncd/triggers.git" - } -}' + -d '{"repository":{"url": "https://github.com/tektoncd/triggers.git"}}' # Ensure the run is successful -kubectl -n $NS wait --for=condition=Succeeded pipelineruns --timeout=120s --all +kubectl -n $TEKTON_NS wait --for=condition=Succeeded pipelineruns --timeout=120s --all # Check the pipeline runs -kubectl -n $NS get pipelinerun +kubectl -n $TEKTON_NS get pipelinerun diff --git a/tools/utils/aio-setup.sh b/tools/utils/aio-setup.sh new file mode 100755 index 00000000..7c8c016d --- /dev/null +++ b/tools/utils/aio-setup.sh @@ -0,0 +1,121 @@ +#!/bin/bash + +set -ex + +: ${TEKTON_NS:="tekton-pipelines"} +: ${HARBOR_NS:="harbor"} +: ${LOKI_NS:="loki-stack"} +: ${NFS_NS:="nfs"} +: ${GRAFANA_NS:="grafana"} +: ${CHART_ROOT_PATH:="./charts"} +: ${CLONE_REPO:="false"} +: ${DEPLOY_K8S:="true"} +: ${HARBOR_VERSION:="1.5.2"} + +# ensures we have git +sudo -E apt install -y git + +# clones upstream rep +if [ $CLONE_REPO == "true" ]; then + git clone "https://review.opendev.org/airship/charts" airship-charts + cd airship-charts +fi + +# deploys k8s locally on machine, this also deploys calico +if [ $DEPLOY_K8S == "true" ]; then + ./tools/gate/deploy-k8s.sh +fi + +# creates namespaces +kubectl create namespace $NFS_NS || true +kubectl create namespace $HARBOR_NS || true +kubectl create namespace $TEKTON_NS || true +kubectl create namespace $GRAFANA_NS || true +kubectl create namespace $LOKI_NS || true + +# adds all the necessary repo here and update +helm repo add osh https://tarballs.opendev.org/openstack/openstack-helm-infra +helm repo add harbor https://helm.goharbor.io +helm repo add loki https://grafana.github.io/loki/charts +helm repo add grafana https://grafana.github.io/helm-charts +helm repo update + +# adds label here so the OSH chart can be target the node +kubectl label nodes --all openstack-control-plane=enabled --overwrite + +# deploys nfs +helm upgrade --install nfs-provisioner osh/nfs-provisioner \ + --namespace=$NFS_NS \ + --set storage.persistentVolumeClaim.size=10Gi \ + --set storage.persistentVolumeClaim.class_name=nfs-provisioner \ + ${EXTRA_HELM_ARGS_NFS} + +# waits for the pods to get ready +kubectl wait --for=condition=ready pod --timeout=600s --all -n $NFS_NS + +# deploys harbor +tee /tmp/harbor.yaml << EOF +expose: + tls: + enabled: false +internalTLS: + enabled: false +persistence: + persistentVolumeClaim: + registry: + storageClass: nfs-provisioner + chartmuseum: + storageClass: nfs-provisioner + jobservice: + storageClass: nfs-provisioner + database: + storageClass: nfs-provisioner + size: 5Gi + redis: + storageClass: nfs-provisioner + trivy: + storageClass: nfs-provisioner +EOF + +helm upgrade --install harbor harbor/harbor \ + --namespace=$HARBOR_NS \ + --values=/tmp/harbor.yaml \ + --version=${HARBOR_VERSION} \ + ${EXTRA_HELM_ARGS_HARBOR} + +# deploys tekton +tee /tmp/dashboard.yaml << EOF +config: + args: + read_only: true +EOF + +helm upgrade --install tekton-pipelines ${CHART_ROOT_PATH}/tekton-pipelines \ + --namespace=$TEKTON_NS \ + ${EXTRA_HELM_ARGS_TEKTON_PIPELINES} + +helm upgrade --install tekton-triggers ${CHART_ROOT_PATH}/tekton-triggers \ + --namespace=$TEKTON_NS \ + ${EXTRA_HELM_ARGS_TEKTON_TRIGGERS} + +helm upgrade --install tekton-dashboard ${CHART_ROOT_PATH}/tekton-dashboard \ + --namespace=$TEKTON_NS \ + --values=/tmp/dashboard.yaml \ + ${EXTRA_HELM_ARGS_TEKTON_DASHBOARD} + +# waits for the pods to get ready +kubectl wait --for=condition=ready pod --timeout=600s --all -n $TEKTON_NS +kubectl wait --for=condition=ready pod --timeout=600s --all -n $HARBOR_NS + +# installs grafana and loki from upstream chart +helm upgrade --install grafana grafana/grafana \ + --namespace=$GRAFANA_NS \ + ${EXTRA_HELM_ARGS_GRAFANA} + +helm upgrade --install loki-stack loki/loki-stack \ + --namespace=$LOKI_NS \ + ${EXTRA_HELM_ARGS_LOKI_STACK} + +# waits for the pods to get ready +kubectl wait --for=condition=ready pod --timeout=600s --all -n $GRAFANA_NS +kubectl wait --for=condition=ready pod --timeout=600s --all -n $LOKI_NS diff --git a/tools/utils/harbor-tekton-aio.sh b/tools/utils/harbor-tekton-aio.sh deleted file mode 100755 index c9d73406..00000000 --- a/tools/utils/harbor-tekton-aio.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -set -ex - -# NOTE(lamt): Runs this script in the root directory of this repo. -: ${NAMESPACE:="tekton-pipelines"} -: ${CHART_ROOT_PATH:="./charts"} - -# deploys a Kubernetes cluster -# ./tools/gate/deploy-k8s.sh - -# creates namespace -kubectl create namespace $NAMESPACE || true - -# TODO(lamt): Needs an PV/C provider - NFS - -# deploys harbor -helm upgrade --install harbor ${CHART_ROOT_PATH}/harbor \ - --namespace=$NAMESPACE \ - ${EXTRA_HELM_ARGS_TEKTON_HARBOR} - -# deploys tekton -helm upgrade --install tekton-pipelines ${CHART_ROOT_PATH}/tekton-pipelines \ - --namespace=$NAMESPACE \ - ${EXTRA_HELM_ARGS_TEKTON_PIPELINES} - -helm upgrade --install tekton-triggers ${CHART_ROOT_PATH}/tekton-triggers \ - --namespace=$NAMESPACE \ - ${EXTRA_HELM_ARGS_TEKTON_TRIGGERS} - -helm upgrade --install tekton-dashboard ${CHART_ROOT_PATH}/tekton-dashboard \ - --namespace=$NAMESPACE \ - ${EXTRA_HELM_ARGS_TEKTON_DASHBOARD} - -# waits for the pods to get ready -kubectl --namespace $NAMESPACE wait --for=condition=ready pod --timeout=600s --all diff --git a/tools/utils/tekton-cleanup.sh b/tools/utils/tekton-cleanup.sh new file mode 100755 index 00000000..d04385a5 --- /dev/null +++ b/tools/utils/tekton-cleanup.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +set -ex + +: ${TEKTON_NS:="tekton-pipelines"} + +tekton_releases=$(helm -n $TEKTON_NS ls -q) +if [ -z "$tekton_releases" ]; then + echo "no release in $TEKTON_NS found" +else + for release in $tekton_releases; do + helm -n $TEKTON_NS uninstall $release + done + + # waits til the resources are cleaned up + sleep 30 + + # helm uninstalls should clean up all the resources, but in the scenarios they are orphaned, + # this should attempt to clean it up. + for resource in mutatingwebhookconfigurations validatingwebhookconfigurations; do + kubectl -n $TEKTON_NS get $resource -o name | awk -F'/' '{print $2}' | grep tekton.dev | xargs -r kubectl -n $TEKTON_NS delete $resource + done +fi diff --git a/zuul.d/jobs.yaml b/zuul.d/jobs.yaml index 274df40d..2a280010 100644 --- a/zuul.d/jobs.yaml +++ b/zuul.d/jobs.yaml @@ -28,10 +28,8 @@ vars: collect_kubernetes_logs_namespace: tekton-pipelines gate_scripts: - - ./tools/gate/tekton/100-deploy-k8s.sh - - ./tools/gate/tekton/200-install.sh - - ./tools/gate/tekton/300-test.sh - - ./tools/gate/harbor/200-install.sh + - ./tools/utils/aio-setup.sh + - ./tools/gate/pipelines/run-test.sh - job: name: airship-charts-upload-git-mirror