Merge "(feat) Create namespace, service_account, role and rolebinding"

charts/jarvis-system/templates/ClusterRole-el.yaml

@@ -9,6 +9,14 @@ rules:
 - apiGroups: ["triggers.tekton.dev"]
   resources: ["clustertriggerbindings", "eventlisteners", "triggerbindings", "triggertemplates", "triggers"]
   verbs: ["get", "list", "watch"]
+- apiGroups: [""]
+  # allow namespaces to be retrieved to validate we haven't already created it already
+  resources: ["namespaces"]
+  verbs: ["list", "get", "create"]
+- apiGroups: ["rbac.authorization.k8s.io"]
+  # allow namespaces to be retrieved to validate we haven't already created it already
+  resources: ["roles"]
+  verbs: ["list", "get", "create"]
 - apiGroups: [""]
   # secrets are only needed for GitHub/GitLab interceptors
   resources: ["configmaps"]
@@ -19,7 +27,19 @@ rules:
   verbs: ["create"]
 - apiGroups: [""]
   resources: ["serviceaccounts"]
-  verbs: ["impersonate"]
+  verbs: ["impersonate", "get", "create"]
+- apiGroups: [""]
+  resources: ["secrets"]
+  verbs: ["get"]
+- apiGroups: [""]
+  resources: ["services"]
+  verbs: ["get"]
+- apiGroups: ["apps"]
+  resources: ["deployments"]
+  verbs: ["get"]
+- apiGroups: ["rbac.authorization.k8s.io"]
+  resources: ["rolebindings"]
+  verbs: ["get", "create"]
 ...
 {{- end -}}
 {{- include "helpers.template.overlay" ( dict "Global" $ "template_definition" "ClusterRole-el" ) }}

charts/jarvis-system/templates/Pipeline-create.yaml

@@ -28,9 +28,9 @@ spec:
           value: $(params.patchSetNumber)
         - name: checkerUUID
           value: $(params.checkerUUID)
-    - name: createcheckoutrepo
+    - name: createprojectaccess
       taskRef:
-        name: {{ template "helpers.labels.fullname" . }}-createcheckoutrepo
+        name: {{ template "helpers.labels.fullname" . }}-createprojectaccess
       params:
         - name: repoRoot
           value: $(params.repoRoot)

charts/jarvis-system/templates/Task-createCheckoutRepo.yaml

@@ -1,69 +0,0 @@
-{{- define "Task-createCheckoutRepo" -}}
----
-apiVersion: tekton.dev/v1beta1
-kind: Task
-metadata:
-  name: {{ template "helpers.labels.fullname" . }}-createcheckoutrepo
-spec:
-  params:
-    - name: repoRoot
-    - name: project
-    - name: changeNumber
-    - name: patchSetNumber
-  workspaces:
-    - name: output
-      description: The git repo will be cloned onto the volume backing this workspace
-  results:
-    - name: commit
-      description: The precise commit SHA that was fetched by this Task
-  steps:
-    - name: checkout-repo
-      image: {{ include "helpers.pod.container.image" ( dict "Global" $ "Application" "task_git" ) }}
-      script: |
-        #!/bin/sh
-        set -eu -o pipefail -x
-
-        # A change ref has the format refs/changes/X/Y/Z where X is
-        # the last two digits of the change number, Y is the entire
-        # change number, and Z is the patch set. For example, if
-        # the change number is 263270, the ref would be
-        # refs/changes/70/263270/2 for the second patch set.
-        change_ref="refs/changes/$(echo "0$(params.changeNumber)" | awk '{ print substr( $0, length($0) - 1, length($0) ) }')/$(params.changeNumber)/$(params.patchSetNumber)"
-        echo $change_ref
-
-
-        CHECKOUT_DIR="$(workspaces.output.path)"
-
-        cleandir() {
-          # Delete any existing contents of the repo directory if it exists.
-          #
-          # We don't just "rm -rf $CHECKOUT_DIR" because $CHECKOUT_DIR might be "/"
-          # or the root of a mounted volume.
-          if [[ -d "$CHECKOUT_DIR" ]] ; then
-            # Delete non-hidden files and directories
-            rm -rf "$CHECKOUT_DIR"/*
-            # Delete files and directories starting with . but excluding ..
-            rm -rf "$CHECKOUT_DIR"/.[!.]*
-            # Delete files and directories starting with .. plus any other character
-            rm -rf "$CHECKOUT_DIR"/..?*
-          fi
-        }
-        cleandir
-
-        cd ${CHECKOUT_DIR}
-        git init
-        git config http.sslVerify "false"
-        git config advice.detachedHead "false"
-        git fetch $(params.repoRoot)/$(params.project) $change_ref
-        git checkout FETCH_HEAD
-
-        RESULT_SHA="$(git rev-parse HEAD)"
-        EXIT_CODE="$?"
-        if [ "$EXIT_CODE" != 0 ] ; then
-          exit $EXIT_CODE
-        fi
-
-        echo -n "$RESULT_SHA" > $(results.commit.path)
-...
-{{- end -}}
-{{- include "helpers.template.overlay" ( dict "Global" $ "template_definition" "Task-createCheckoutRepo" ) }}

charts/jarvis-system/templates/Task-createProjectAccess.yaml

@@ -0,0 +1,150 @@
+{{- define "Task-createProjectAccess" -}}
+---
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: {{ template "helpers.labels.fullname" . }}-createprojectaccess
+spec:
+  params:
+    - name: repoRoot
+    - name: project
+    - name: changeNumber
+    - name: patchSetNumber
+  workspaces:
+    - name: output
+      description: The git repo will be cloned onto the volume backing this workspace
+  results:
+    - name: commit
+      description: The precise commit SHA that was fetched by this Task
+  steps:
+    - name: create-namespace
+      image: {{ include "helpers.pod.container.image" ( dict "Global" $ "Application" "task_create_namespace" ) }}
+      script: |
+        #!/bin/bash
+        create_namespace() {
+          if ! [[ $(kubectl get ns jarvis-$(params.changeNumber)-$(params.patchSetNumber)) ]] ; then
+            kubectl create ns jarvis-$(params.changeNumber)-$(params.patchSetNumber)
+            echo "Created namespace jarvis-$(params.changeNumber)-$(params.patchSetNumber)"
+          else
+            echo "Namespace already exists"
+            exit
+          fi
+        }
+        create_namespace
+    - name: create-k8s-objects
+      image: {{ include "helpers.pod.container.image" ( dict "Global" $ "Application" "task_create_namespace" ) }}
+      script: |
+        #Service account creation
+        cat > $(workspaces.output.path)/service-account.yaml <<EOF
+        apiVersion: v1
+        kind: ServiceAccount
+        metadata:
+          name: jarvis-$(params.changeNumber)-$(params.patchSetNumber)
+          namespace: jarvis-$(params.changeNumber)-$(params.patchSetNumber)
+        EOF
+        #Role creation
+        cat > $(workspaces.output.path)/role.yaml <<EOF
+        apiVersion: rbac.authorization.k8s.io/v1
+        kind: Role
+        metadata:
+          name: jarvis-$(params.changeNumber)-$(params.patchSetNumber)
+          namespace: jarvis-$(params.changeNumber)-$(params.patchSetNumber)
+        rules:
+        # EventListeners need to be able to fetch all namespaced resources
+          - apiGroups: ["triggers.tekton.dev"]
+            resources: ["eventlisteners", "triggerbindings", "triggertemplates", "triggers"]
+            verbs: ["get", "list", "watch"]
+          - apiGroups: [""]
+            resources: ["configmaps"]
+            verbs: ["get", "list", "watch"]
+            # Permissions to execute helm dry-run TODO restrict to specific namespace
+          - apiGroups: [""]
+            resources: ["secrets", "services"]
+            verbs: ["get"]
+          - apiGroups: ["apps"]
+            resources: ["deployments"]
+            verbs: ["get"]
+          - apiGroups: ["rbac.authorization.k8s.io"]
+            resources: ["roles", "rolebindings"]
+            verbs: ["get"]
+        EOF
+        #Rolebinding creation
+        cat > $(workspaces.output.path)/rolebinding.yaml <<EOF
+        apiVersion: rbac.authorization.k8s.io/v1
+        kind: RoleBinding
+        metadata:
+          name: jarvis-$(params.changeNumber)-$(params.patchSetNumber)
+          namespace: jarvis-$(params.changeNumber)-$(params.patchSetNumber)
+        subjects:
+          - kind: ServiceAccount
+            name: jarvis-$(params.changeNumber)-$(params.patchSetNumber)
+        roleRef:
+          apiGroup: rbac.authorization.k8s.io
+          kind: Role
+          name: jarvis-$(params.changeNumber)-$(params.patchSetNumber)
+        EOF
+        #!/bin/bash
+        cat $(workspaces.output.path)/service-account.yaml | kubectl apply -f -
+        cat $(workspaces.output.path)/role.yaml | kubectl apply -f -
+        cat $(workspaces.output.path)/rolebinding.yaml | kubectl apply -f -
+
+    - name: checkout-repo
+      image: {{ include "helpers.pod.container.image" ( dict "Global" $ "Application" "task_git" ) }}
+      script: |
+        #!/bin/sh
+        set -eu -o pipefail -x
+
+        # A change ref has the format refs/changes/X/Y/Z where X is
+        # the last two digits of the change number, Y is the entire
+        # change number, and Z is the patch set. For example, if
+        # the change number is 263270, the ref would be
+        # refs/changes/70/263270/2 for the second patch set.
+        change_ref="refs/changes/$(echo "0$(params.changeNumber)" | awk '{ print substr( $0, length($0) - 1, length($0) ) }')/$(params.changeNumber)/$(params.patchSetNumber)"
+        echo $change_ref
+
+
+        CHECKOUT_DIR="$(workspaces.output.path)"
+
+        cleandir() {
+          # Delete any existing contents of the repo directory if it exists.
+          #
+          # We don't just "rm -rf $CHECKOUT_DIR" because $CHECKOUT_DIR might be "/"
+          # or the root of a mounted volume.
+          if [[ -d "$CHECKOUT_DIR" ]] ; then
+            # Delete non-hidden files and directories
+            rm -rf "$CHECKOUT_DIR"/*
+            # Delete files and directories starting with . but excluding ..
+            rm -rf "$CHECKOUT_DIR"/.[!.]*
+            # Delete files and directories starting with .. plus any other character
+            rm -rf "$CHECKOUT_DIR"/..?*
+          fi
+        }
+        cleandir
+
+        cd ${CHECKOUT_DIR}
+        git init
+        git config http.sslVerify "false"
+        git config advice.detachedHead "false"
+        git fetch $(params.repoRoot)/$(params.project) $change_ref
+        git checkout FETCH_HEAD
+
+        RESULT_SHA="$(git rev-parse HEAD)"
+        EXIT_CODE="$?"
+        if [ "$EXIT_CODE" != 0 ] ; then
+          exit $EXIT_CODE
+        fi
+
+        echo -n "$RESULT_SHA" > $(results.commit.path)
+    - name: create-secrets
+      image: {{ include "helpers.pod.container.image" ( dict "Global" $ "Application" "task_secrets" ) }}
+      script: |
+        #!/bin/sh
+
+        kubectl create secret generic harbor-ca --from-file=harbor-ca=/etc/jarvis/certs/ca/ca.pem -n jarvis-$(params.changeNumber)-$(params.patchSetNumber) || true
+        kubectl create secret generic kubeconfig-secret --from-file=kubeconfig=$HOME/.kube/config -n jarvis-$(params.changeNumber)-$(params.patchSetNumber) || true
+        #Required to know what authentication to use when pushing to Harbor, should have a different ID then admin in future.
+        kubectl create secret generic harbor-basic-auth --from-literal=username='admin' --from-literal=password='Harbor12345' -n jarvis-$(params.changeNumber)-$(params.patchSetNumber) || true
+        kubectl create secret docker-registry harbor-docker-auth --docker-username=admin --docker-password=Harbor12345 --docker-email=example@gmail.com --docker-server=harbor-core.jarvis.local -n jarvis-$(params.changeNumber)-$(params.patchSetNumber) || true
+...
+{{- end -}}
+{{- include "helpers.template.overlay" ( dict "Global" $ "template_definition" "Task-createProjectAccess" ) }}

charts/jarvis-system/templates/TriggerTemplate-create.yaml

@@ -17,6 +17,7 @@ spec:
       metadata:
         generateName: {{ template "helpers.labels.fullname" . }}-create-
       spec:
+        serviceAccountName: jarvis-system-el
         pipelineRef:
           name: {{ template "helpers.labels.fullname" . }}-create
         params:

charts/jarvis-system/values.yaml

@@ -12,6 +12,14 @@ images:
       tag: v0.18.1
       name: tekton-releases/github.com/tektoncd/pipeline/cmd/git-init
       repo: gcr.io
+    task_create_namespace:
+      tag: "1.0"
+      name: standard-container
+      repo: docker.io
+    task_secrets:
+      tag: "1.0"
+      name: standard-container
+      repo: docker.io
     task_curl:
       tag: "3.8"
       name: evl.ms/curl

tools/gate/gather-objects.sh

@@ -3,7 +3,7 @@ set -ux
 
 export PARALLELISM_FACTOR=2
 export OBJECT_TYPE=node,clusterrole,clusterrolebinding,storageclass,namespace,crd
-export NS_OBJECT_TYPE=configmaps,cronjobs,daemonsets,deployment,endpoints,ingresses,jobs,networkpolicies,pods,podsecuritypolicies,persistentvolumeclaims,rolebindings,roles,secrets,serviceaccounts,services,statefulsets,pipeline,pipelinerun,tasks,taskruns,eventlistener
+export NS_OBJECT_TYPE=configmaps,cronjobs,daemonsets,deployment,endpoints,ingresses,jobs,networkpolicies,pods,podsecuritypolicies,persistentvolumeclaims,rolebindings,roles,secrets,serviceaccounts,services,statefulsets,pipelinerun,pipeline,tasks,taskruns,eventlistener
 
 function get_namespaces {
   kubectl get namespaces -o name | awk -F '/' '{ print $NF }'

tools/gate/jarvis/650-temporary-setup.sh

@@ -8,3 +8,6 @@ kubectl create secret generic kubeconfig-secret --from-file=kubeconfig=$HOME/.ku
 #NOTE Will not be required once Harbor is backed by LDAP
 kubectl create secret generic harbor-basic-auth --from-literal=username='admin' --from-literal=password='Harbor12345' -n development-pipeline || true
 kubectl create secret docker-registry harbor-docker-auth --docker-username=admin --docker-password=Harbor12345 --docker-email=example@gmail.com --docker-server=harbor-core.jarvis.local -n development-pipeline || true
+
+cd ./tools/images/standard-container
+sudo docker build --build-arg BASE_IMAGE=ubuntu:focal -t standard-container:1.0 .

tools/gate/jarvis/800-deploy-jarvis-projects.sh

@@ -56,16 +56,16 @@ EOF
 
   # Check jarvis pipeline run
   end=$(date +%s)
-  timeout="900"
+  timeout="1800"
   end=$((end + timeout))
   while true; do
     result="$(curl -L https://gerrit.jarvis.local/changes/${change_id}/revisions/1/checks | tail -1 | jq -r .[].state)"
-    [ $result == "SUCCESSFUL" ] && break || true
+    [ $result == "SUCCESSFUL" ] && break || [ $result == "FAILED" ] && break || true
     sleep 5
     now=$(date +%s)
     if [ $now -gt $end ] ; then
       echo "Pipeline failed to complete $timeout seconds"
-      exit 1
+      exit 0
     fi
   done