Gerrit: ldap backed auth, verified votes, and webhook plugin

This PS updates Gerrit to use LDAP for credential management
permitting both easier CI by permitting HTTP basic auth for API
requests, and closer matching a real world config.

Verified labeled votes are also enabled and configured, for CI to
respond to.

Additionally the basis of using gerrit to trigger tekton pipelines
is validated.

As a rapid follow on to this, param passing from the webhook to the
trigger will need to be fleshed out.

Change-Id: Id135c71c7f94df14e5583b5ceb7db25688f89dfe
Signed-off-by: Pete Birley <pete@port.direct>

charts/Makefile

@@ -16,7 +16,7 @@ TASK  := build
 
 EXCLUDES := playbooks roles doc tests tools logs tmp zuul.d releasenotes
 # NOTE(Portdirect): Exlude chart directories that are presnt only for the purposes of housing over-rides
-EXCLUDES += harbor loki gerrit
+EXCLUDES += harbor loki gerrit ldap
 CHARTS := $(filter-out $(EXCLUDES), $(patsubst %/.,%,$(wildcard */.)))
 
 .PHONY: $(EXCLUDES) $(CHARTS)

charts/gerrit/values_overrides/default.yaml

@@ -17,6 +17,14 @@ ingress:
       name: gerrit-tls
 
 gerrit:
+  plugins:
+    packaged:
+      - commit-message-length-validator
+      - download-commands
+      - replication
+      - reviewnotes
+      - webhooks
+      - delete-project
   service:
     ssh:
       enabled: true
@@ -42,7 +50,18 @@ gerrit:
           type = LUCENE
           onlineUpgrade = false # FIXED
         [auth]
-          type = DEVELOPMENT_BECOME_ANY_ACCOUNT
+          type = LDAP
+          gitBasicAuthPolicy = LDAP
+        [ldap]
+          server = ldap://ldap-openldap.ldap.svc.cluster.local
+          username = cn=readonly,dc=jarvis,dc=local
+          password = readonly
+          accountBase = ou=Users,dc=jarvis,dc=local
+          accountPattern = (&(objectClass=inetOrgPerson)(uid=${username}))
+          accountFullName = ${cn}
+          groupBase = ou=Groups,dc=jarvis,dc=local
+          groupMemberPattern = (member=${dn})
+          localUsernameToLowerCase = true
         [httpd]
           # If using an ingress use proxy-http or proxy-https
           listenUrl = proxy-http://*:8080/

charts/ldap/values_overrides/default.yaml

@@ -0,0 +1,39 @@
+env:
+  LDAP_ORGANISATION: Jarvis CICD
+  LDAP_DOMAIN: jarvis.local
+  LDAP_BASE_DN: dc=jarvis,dc=local
+  LDAP_READONLY_USER: "true"
+  LDAP_READONLY_USER_USERNAME: readonly
+  LDAP_READONLY_USER_PASSWORD: readonly
+  LDAP_ADMIN_PASSWORD: admin
+  LDAP_TLS: "false"
+  LDAP_TLS_ENFORCE: "false"
+  LDAP_REMOVE_CONFIG_AFTER_SETUP: "false"
+customLdifFiles:
+  groups.ldif: |-
+    dn: ou=Users,dc=jarvis,dc=local
+    changetype: add
+    objectClass: organizationalUnit
+    ou: Users
+
+    dn: ou=Groups,dc=jarvis,dc=local
+    changetype: add
+    objectClass: organizationalUnit
+    ou: Groups
+
+    dn: uid=jarvis,ou=Users,dc=jarvis,dc=local
+    changetype: add
+    objectClass: top
+    objectClass: person
+    objectClass: inetOrgPerson
+    cn: jarvis
+    sn: User
+    displayname: jarvis User
+    mail: jarvis@cluster.local
+    userpassword: {SSHA}fCJ5vuW1BQ4/OfOVkkx1qjwi7yHFuGNB
+
+    dn: cn=jarvis-admins,ou=Groups,dc=jarvis,dc=local
+    changetype: add
+    objectClass: top
+    objectClass: groupOfNames
+    member: uid=jarvis,ou=Users,dc=jarvis,dc=local

tools/gate/deploy-k8s.sh

@@ -101,7 +101,8 @@ sudo -E apt-get install -y \
   libffi-dev \
   ipvsadm \
   make \
-  bc
+  bc \
+  git-review
 
 # Prepare tmpfs for etcd
 sudo mkdir -p /data

tools/gate/jarvis/500-deploy-gerrit.sh

@@ -1,6 +1,18 @@
 #!/bin/bash
 set -ex
 
+helm repo add stable https://charts.helm.sh/stable
+# shellcheck disable=SC2046
+helm upgrade \
+    --create-namespace \
+    --install \
+    --namespace=ldap \
+    ldap \
+    stable/openldap \
+    $(./tools/deployment/common/get-values-overrides.sh ldap)
+
+./tools/deployment/common/wait-for-pods.sh ldap
+
 gerrit_source=$(mktemp -d)
 repo_sha="251041b192ef8acf1963d747482126d0e9e66e75"
 repo_remote="https://gerrit.googlesource.com/k8s-gerrit"
@@ -58,4 +70,71 @@ kubectl patch -n gerrit svc gerrit-gerrit-service --patch '{
       }
     ]
   }
-}'
+}'
+
+function gerrit_bootstrap() {
+  # Define creds to use for gerrit.
+  ldap_username="jarvis"
+  ldap_password="password"
+
+  # Login to gerrit, to provision admin account
+  curl --verbose \
+    -d "username=${ldap_username}&password=${ldap_password}" \
+    -X POST \
+    https://gerrit.jarvis.local/login
+
+  # Create SSH Keys if the private key does not already exist, note this will fail if a public key already
+  # exists at the default location without a corresponding private key.
+  mkdir -p "${HOME}/.ssh"
+  if [[ ! -f "${HOME}/.ssh/id_rsa" ]]; then
+    ssh-keygen -t rsa -f "${HOME}/.ssh/id_rsa" -q -N ""
+  fi
+
+  # Add SSH Public key to gerrit
+  curl --verbose \
+    -u "${ldap_username}:${ldap_password}" \
+    -X POST \
+    -H "Content-Type: text/plain" \
+    --data "@${HOME}/.ssh/id_rsa.pub" \
+    https://gerrit.jarvis.local/a/accounts/self/sshkeys/
+
+  # Add Gerrit HostKey to SSH known hosts
+  ssh-keyscan -p 29418 -H gerrit.jarvis.local >> "${HOME}/.ssh/known_hosts"
+
+  # Validate access to gerrit via SSH
+  ssh -p 29418 ${ldap_username}@gerrit.jarvis.local gerrit version
+
+  # Configure Git
+  git config --global user.name "Edwin Jarvis"
+  git config --global user.email "jarvis@cluster.local"
+  git config --global --add gitreview.username "jarvis"
+
+  # Clone, fetch and checkout project config repo
+  all_projects_repo=$(mktemp -d)
+  git clone ssh://${ldap_username}@gerrit.jarvis.local:29418/All-Projects.git "${all_projects_repo}"
+  pushd "${all_projects_repo}"
+  git fetch origin refs/meta/config:refs/remotes/origin/meta/config
+  git checkout meta/config
+
+  # Configure Verified Label
+  tee --append project.config <<EOF
+[label "Verified"]
+        function = MaxWithBlock
+        defaultValue = 0
+        value = -1 Fails
+        value = 0 No score
+        value = +1 Verified
+        copyAllScoresIfNoCodeChange = true
+EOF
+
+  # Give Admins, Service Users and Project Owners voting rights for the Verified Label
+  sed -i '/\[access "refs\/heads\/\*"\]/a\ \ \ \ \ \ \ \ label-Verified = -1..+1 group Administrators\n\ \ \ \ \ \ \ \ label-Verified = -1..+1 group Service Users\n\ \ \ \ \ \ \ \ label-Verified = -1..+1 group Project Owners' project.config
+
+  # Commit and push config
+  git add .
+  git commit -asm "Create Verified Label"
+  git push origin HEAD:refs/meta/config
+  popd
+}
+
+gerrit_bootstrap

tools/gate/jarvis/600-deploy-tekton.sh

@@ -30,16 +30,58 @@ function validate() {
   kubectl -n tekton-pipelines apply -f ./tools/gate/jarvis/resources/tekton/yaml/example-pipeline.yaml
   kubectl -n tekton-pipelines wait --for=condition=Ready pod --timeout=120s --all
 
-  kubectl get po -A
+  # Define creds to use for gerrit.
+  ldap_username="jarvis"
+  ldap_password="password"
 
-  # Trigger the sample github pipeline
-  local listener_ip
-  listener_ip="$(kubectl -n tekton-pipelines get svc el-listener -o 'jsonpath={.spec.clusterIP}')"
-  curl -X POST \
-    "http://${listener_ip}:8080" \
-    -H 'Content-Type: application/json' \
-    -H 'X-Hub-Signature: sha1=2da37dcb9404ff17b714ee7a505c384758ddeb7b' \
-    -d '{"repository":{"url": "https://github.com/tektoncd/triggers.git"}}'
+  # Create repo for Jarvis sanity testing
+  ssh -p 29418 ${ldap_username}@gerrit.jarvis.local gerrit create-project jarvis-sanity --submit-type MERGE_IF_NECESSARY --owner Administrators --empty-commit
+
+  # Configure repo webhook
+  jarvis_sanity_repo=$(mktemp -d)
+  pushd "${jarvis_sanity_repo}"
+  git init
+  git remote add origin ssh://${ldap_username}@gerrit.jarvis.local:29418/jarvis-sanity.git
+  git fetch origin refs/meta/config:refs/remotes/origin/meta/config
+  git checkout meta/config
+  tee --append project.config <<EOF
+[plugin "webhooks"]
+    connectionTimeout = 3000
+    maxTries = 300
+    retryInterval = 2000
+    socketTimeout = 2500
+    threadPoolSize = 3
+EOF
+  tee webhooks.config <<EOF
+[remote "Tekton"]
+    url = http://el-listener.tekton-pipelines.svc.cluster.local:8080/
+    maxTries = 3
+    sslVerify = false
+    event = patchset-created
+    event = patchset-updated
+EOF
+  git add .
+  git commit -asm "Create Add PS Webhook Config"
+  git push origin HEAD:refs/meta/config
+  popd
+
+  # Create PS to repo to test webhook
+  jarvis_sanity_repo=$(mktemp -d)
+  git clone ssh://${ldap_username}@gerrit.jarvis.local:29418/jarvis-sanity.git "${jarvis_sanity_repo}"
+  pushd "${jarvis_sanity_repo}"
+  tee .gitreview <<EOF
+[gerrit]
+host=gerrit.jarvis.local
+port=29418
+project=jarvis-sanity.git
+EOF
+  git review -s
+  git add .gitreview
+  git commit -asm "Add .gitreview"
+  git review
+
+  # Sleep for 5s to give time for webhook to fire, and be responded to
+  sleep 5
 
   # Ensure the run is successful
   kubectl -n tekton-pipelines wait --for=condition=Succeeded pipelineruns --timeout=120s --all

tools/gate/jarvis/resources/tekton/yaml/triggerbindings/triggerbinding.yaml

@@ -4,9 +4,11 @@ metadata:
   name: pipeline-binding
 spec:
   params:
+  #TODO: These have to come from the gerrit trigger.
   - name: gitrevision
-    value: $(body.head_commit.id)
+    value: master
   - name: gitrepositoryurl
-    value: $(body.repository.url)
+    value: https://review.opendev.org/airship/charts.git
   - name: contenttype
-    value: $(header.Content-Type)
+    value: application/json
+