Create secrets from project's jarvis.yaml

Each CNF project now has a jarvis.yaml defining credentials for
harbor and docker, kubeconfig, and certs. The existing ldap
users are used in the jarvis.yaml for both the staging and
non-staging phases of the CI.

The kubeconfig and harbor-ca entries of the jarvis.yaml are
populated during the 800 script and adds those entries as
base64 encoded data so that the required format (indentations and
spacing) is maintained. The Task-createProjectAccess then decodes
the kubeconfig and harbor-ca and creates the secrets for them in
the correct namespace. Secret creation all takes place in the
Task-createProjectAccess.

Change-Id: If0c243416323e36a6f7797d8d378961552193c0d

charts/jarvis-project/templates/Job-project.yaml

@@ -83,7 +83,7 @@ spec:
                   --netrc-file /run/jarvis/harbor-netrc/harbor-netrc \
                   -H "accept: application/json" \
                   -H "Content-Type: application/json" \
-                  -d "{ \"project_name\": \"${PROJECT_NAME}\", \"public\": true, \"metadata\": { \"auto_scan\": \"true\" }}"` ;;
+                  -d "{ \"project_name\": \"${PROJECT_NAME}\", \"public\": false, \"metadata\": { \"auto_scan\": \"true\" }}"` ;;
                   *) echo "Unable to retrieve projects in harbor: ${PROJECT_CHECK_RESP} response code."; exit 1
                   esac
                 done

charts/jarvis-system/templates/Task-createProjectAccess.yaml

@@ -171,35 +171,52 @@ spec:
       script: |
         #!/bin/bash
         ## Creating kubeconfig secret in correct namespace
-        SECRET_JSON_ORIGINAL=$(mktemp --suffix=".json")
+        CHECKOUT_DIR="$(workspaces.output.path)"
-        kubectl get secret -n development-pipeline kubeconfig-secret -o=json > "$SECRET_JSON_ORIGINAL"
+        # base64 decode the kubeconfig as it was previously encoded to preserve the needed formatting
-        SECRET_JSON=$(mktemp --suffix=".json")
+        KUBECONFIG_DECODED=$(yq -r '.dev."jarvis-aio".kubeconfig' ${CHECKOUT_DIR}/jarvis.yaml | base64 --decode)
-        jq 'del(.metadata.namespace) | del(.metadata.creationTimestamp) | del(.metadata.labels."controller-uid") | del(.metadata.resourceVersion) | del(.metadata.selfLink) | del(.metadata.uid) | del(.spec.selector) | del(.spec.template.metadata.creationTimestamp) | del(.spec.template.metadata.labels."controller-uid" )' "$SECRET_JSON_ORIGINAL" > "$SECRET_JSON"
+        kubectl create secret generic kubeconfig-secret -n jarvis-$(params.changeNumber)-$(params.patchSetNumber) --from-literal=kubeconfig="$KUBECONFIG_DECODED"
-
-        cat "$SECRET_JSON" | kubectl create -n jarvis-$(params.changeNumber)-$(params.patchSetNumber) -f -
 
         ## Creating Harbor certificate in correct namespace
-        SECRET_JSON_ORIGINAL=$(mktemp --suffix=".json")
+        # base64 decode the ca as it was previously encoded to preserve the needed formatting
-        kubectl get secret -n development-pipeline harbor-ca -o=json > "$SECRET_JSON_ORIGINAL"
+        CRT_DECODED=$(yq -r '.dev."jarvis-aio"."harbor-ca"' ${CHECKOUT_DIR}/jarvis.yaml | base64 --decode)
-        SECRET_JSON=$(mktemp --suffix=".json")
+        kubectl create secret generic harbor-ca -n jarvis-$(params.changeNumber)-$(params.patchSetNumber) --from-literal=harbor-ca="$CRT_DECODED"
-        jq 'del(.metadata.namespace) | del(.metadata.creationTimestamp) | del(.metadata.labels."controller-uid") | del(.metadata.resourceVersion) | del(.metadata.selfLink) | del(.metadata.uid) | del(.spec.selector) | del(.spec.template.metadata.creationTimestamp) | del(.spec.template.metadata.labels."controller-uid" )' "$SECRET_JSON_ORIGINAL" > "$SECRET_JSON"
 
-        cat "$SECRET_JSON" | kubectl create -n jarvis-$(params.changeNumber)-$(params.patchSetNumber) -f -
+        # Harbor basic auth and docker auth
-
+        HELM_USERNAME=$(yq -r '.dev."jarvis-aio"."helm-staging".username' ${CHECKOUT_DIR}/jarvis.yaml)
-        rm "$SECRET_JSON_ORIGINAL"
+        HELM_PASSWORD=$(yq -r '.dev."jarvis-aio"."helm-staging".password' ${CHECKOUT_DIR}/jarvis.yaml)
-        rm "$SECRET_JSON"
-
-        ## Harbor basic auth and docker auth
         kubectl create secret generic harbor-basic-auth \
-          --from-literal=username='admin' \
+          --from-literal=username=$HELM_USERNAME \
-          --from-literal=password='Harbor12345' \
+          --from-literal=password=$HELM_PASSWORD \
           -n jarvis-$(params.changeNumber)-$(params.patchSetNumber)
 
+        HELM_USERNAME=$(yq -r '.dev."jarvis-aio".helm.username' ${CHECKOUT_DIR}/jarvis.yaml)
+        HELM_PASSWORD=$(yq -r '.dev."jarvis-aio".helm.password' ${CHECKOUT_DIR}/jarvis.yaml)
+        kubectl create secret generic harbor-basic-auth-promote \
+          --from-literal=username=$HELM_USERNAME \
+          --from-literal=password=$HELM_PASSWORD \
+          -n jarvis-$(params.changeNumber)-$(params.patchSetNumber)
+
+        HARBOR_USERNAME=$(yq -r '.dev."jarvis-aio"."harbor-staging".username' ${CHECKOUT_DIR}/jarvis.yaml)
+        HARBOR_PASSWORD=$(yq -r '.dev."jarvis-aio"."harbor-staging".password' ${CHECKOUT_DIR}/jarvis.yaml)
+        HARBOR_EMAIL=$(yq -r '.dev."jarvis-aio"."harbor-staging".email' ${CHECKOUT_DIR}/jarvis.yaml)
+        HARBOR_SERVER=$(yq -r '.dev."jarvis-aio"."harbor-staging".server' ${CHECKOUT_DIR}/jarvis.yaml)
         kubectl create secret docker-registry harbor-docker-auth \
-          --docker-username=admin \
+          --docker-username=$HARBOR_USERNAME \
-          --docker-password=Harbor12345 \
+          --docker-password=$HARBOR_PASSWORD \
-          --docker-email=example@gmail.com \
+          --docker-email=$HARBOR_EMAIL \
-          --docker-server=harbor-core.jarvis.local \
+          --docker-server=$HARBOR_SERVER \
+          -n jarvis-$(params.changeNumber)-$(params.patchSetNumber)
+
+        # Need this new secret to be able to 'promote-artifacts'
+        HARBOR_USERNAME=$(yq -r '.dev."jarvis-aio".harbor.username' ${CHECKOUT_DIR}/jarvis.yaml)
+        HARBOR_PASSWORD=$(yq -r '.dev."jarvis-aio".harbor.password' ${CHECKOUT_DIR}/jarvis.yaml)
+        HARBOR_EMAIL=$(yq -r '.dev."jarvis-aio".harbor.email' ${CHECKOUT_DIR}/jarvis.yaml)
+        HARBOR_SERVER=$(yq -r '.dev."jarvis-aio".harbor.server' ${CHECKOUT_DIR}/jarvis.yaml)
+        kubectl create secret docker-registry harbor-docker-auth-promote \
+          --docker-username=$HARBOR_USERNAME \
+          --docker-password=$HARBOR_PASSWORD \
+          --docker-email=$HARBOR_EMAIL \
+          --docker-server=$HARBOR_SERVER \
           -n jarvis-$(params.changeNumber)-$(params.patchSetNumber)
     - name: install-development-pipeline
       image: {{ include "helpers.pod.container.image" ( dict "Global" $ "Application" "task_secrets" ) }}

tools/gate/jarvis/5G-SA-core/amf/jarvis.yaml

@@ -0,0 +1,121 @@
+dev:
+  jarvis-aio:
+    # Left blank as the kubeconfig is added in during execution of the 800-deploy-jarvis-projects.sh script
+    # The kubeconfig is added in from ~/.kube/config
+    kubeconfig:
+    # Left blank as the harbor-ca is added in during execution of the 800-deploy-jarvis-projects.sh script
+    # The harbor-ca is added in from /etc/jarvis/certs/ca/ca.pem
+    harbor-ca:
+    helm:
+      username: amf-harbor
+      password: harbor-user-password
+    helm-staging:
+      username: amf-harbor-staging
+      password: harbor-staging-user-password
+    harbor:
+      username: amf-harbor
+      password: harbor-user-password
+      email: example@gmail.com
+      server: harbor-core.jarvis.local
+    harbor-staging:
+      username: amf-harbor-staging
+      password: harbor-staging-user-password
+      email: example@gmail.com
+      server: harbor-core.jarvis.local
+#   clusterA:
+#     kubeconfig:
+#     harbor-ca:
+#     helm:
+#       username:
+#       password:
+#     helm-staging:
+#       username:
+#       password:
+#     harbor:
+#       username:
+#       password:
+#       email:
+#       server:
+#     harbor-staging:
+#       username:
+#       password:
+#       email:
+#       server:
+# sil:
+#   clusterA:
+#     kubeconfig:
+#     harbor-ca:
+#     helm:
+#       username:
+#       password:
+#     helm-staging:
+#       username:
+#       password:
+#     harbor:
+#       username:
+#       password:
+#       email:
+#       server:
+#     harbor-staging:
+#       username:
+#       password:
+#       email:
+#       server:
+#   clusterB:
+#     kubeconfig:
+#     harbor-ca:
+#     helm:
+#       username:
+#       password:
+#     helm-staging:
+#       username:
+#       password:
+#     harbor:
+#       username:
+#       password:
+#       email:
+#       server:
+#     harbor-staging:
+#       username:
+#       password:
+#       email:
+#       server:
+# prod:
+#   clusterA:
+#     kubeconfig:
+#     harbor-ca:
+#     helm:
+#       username:
+#       password:
+#     helm-staging:
+#       username:
+#       password:
+#     harbor:
+#       username:
+#       password:
+#       email:
+#       server:
+#     harbor-staging:
+#       username:
+#       password:
+#       email:
+#       server:
+#   clusterB:
+#     kubeconfig:
+#     harbor-ca:
+#     helm:
+#       username:
+#       password:
+#     helm-staging:
+#       username:
+#       password:
+#     harbor:
+#       username:
+#       password:
+#       email:
+#       server:
+#     harbor-staging:
+#       username:
+#       password:
+#       email:
+#       server:

tools/gate/jarvis/5G-SA-core/amf/jarvis/development-pipeline/templates/config_map.yaml

@@ -73,6 +73,8 @@ data:
       "chart_repository": "jarvis-harbor",
       "docker_registry": "harbor-core.jarvis.local",
       "harbor_secret_mounted_path": "/workspace/helm-creds",
+      "harbor_secret_pull_mounted_path": "/workspace/helm-creds/pull",
+      "harbor_secret_push_mounted_path": "/workspace/helm-creds/push",
       "clone_async_batch_size": 20,
       "chart_async_batch_size": 20,
       "image_async_batch_size": 2

tools/gate/jarvis/5G-SA-core/ausf/jarvis.yaml

@@ -0,0 +1,121 @@
+dev:
+  jarvis-aio:
+    # Left blank as the kubeconfig is added in during execution of the 800-deploy-jarvis-projects.sh script
+    # The kubeconfig is added in from ~/.kube/config
+    kubeconfig:
+    # Left blank as the harbor-ca is added in during execution of the 800-deploy-jarvis-projects.sh script
+    # The harbor-ca is added in from /etc/jarvis/certs/ca/ca.pem
+    harbor-ca:
+    helm:
+      username: ausf-harbor
+      password: harbor-user-password
+    helm-staging:
+      username: ausf-harbor-staging
+      password: harbor-staging-user-password
+    harbor:
+      username: ausf-harbor
+      password: harbor-user-password
+      email: example@gmail.com
+      server: harbor-core.jarvis.local
+    harbor-staging:
+      username: ausf-harbor-staging
+      password: harbor-staging-user-password
+      email: example@gmail.com
+      server: harbor-core.jarvis.local
+#   clusterA:
+#     kubeconfig:
+#     harbor-ca:
+#     helm:
+#       username:
+#       password:
+#     helm-staging:
+#       username:
+#       password:
+#     harbor:
+#       username:
+#       password:
+#       email:
+#       server:
+#     harbor-staging:
+#       username:
+#       password:
+#       email:
+#       server:
+# sil:
+#   clusterA:
+#     kubeconfig:
+#     harbor-ca:
+#     helm:
+#       username:
+#       password:
+#     helm-staging:
+#       username:
+#       password:
+#     harbor:
+#       username:
+#       password:
+#       email:
+#       server:
+#     harbor-staging:
+#       username:
+#       password:
+#       email:
+#       server:
+#   clusterB:
+#     kubeconfig:
+#     harbor-ca:
+#     helm:
+#       username:
+#       password:
+#     helm-staging:
+#       username:
+#       password:
+#     harbor:
+#       username:
+#       password:
+#       email:
+#       server:
+#     harbor-staging:
+#       username:
+#       password:
+#       email:
+#       server:
+# prod:
+#   clusterA:
+#     kubeconfig:
+#     harbor-ca:
+#     helm:
+#       username:
+#       password:
+#     helm-staging:
+#       username:
+#       password:
+#     harbor:
+#       username:
+#       password:
+#       email:
+#       server:
+#     harbor-staging:
+#       username:
+#       password:
+#       email:
+#       server:
+#   clusterB:
+#     kubeconfig:
+#     harbor-ca:
+#     helm:
+#       username:
+#       password:
+#     helm-staging:
+#       username:
+#       password:
+#     harbor:
+#       username:
+#       password:
+#       email:
+#       server:
+#     harbor-staging:
+#       username:
+#       password:
+#       email:
+#       server:

tools/gate/jarvis/5G-SA-core/ausf/jarvis/development-pipeline/templates/config_map.yaml

@@ -73,6 +73,8 @@ data:
       "chart_repository": "jarvis-harbor",
       "docker_registry": "harbor-core.jarvis.local",
       "harbor_secret_mounted_path": "/workspace/helm-creds",
+      "harbor_secret_pull_mounted_path": "/workspace/helm-creds/pull",
+      "harbor_secret_push_mounted_path": "/workspace/helm-creds/push",
       "clone_async_batch_size": 20,
       "chart_async_batch_size": 20,
       "image_async_batch_size": 2

tools/gate/jarvis/5G-SA-core/mongodb/jarvis.yaml

@@ -0,0 +1,121 @@
+dev:
+  jarvis-aio:
+    # Left blank as the kubeconfig is added in during execution of the 800-deploy-jarvis-projects.sh script
+    # The kubeconfig is added in from ~/.kube/config
+    kubeconfig:
+    # Left blank as the harbor-ca is added in during execution of the 800-deploy-jarvis-projects.sh script
+    # The harbor-ca is added in from /etc/jarvis/certs/ca/ca.pem
+    harbor-ca:
+    helm:
+      username: mongodb-harbor
+      password: harbor-user-password
+    helm-staging:
+      username: mongodb-harbor-staging
+      password: harbor-staging-user-password
+    harbor:
+      username: mongodb-harbor
+      password: harbor-user-password
+      email: example@gmail.com
+      server: harbor-core.jarvis.local
+    harbor-staging:
+      username: mongodb-harbor-staging
+      password: harbor-staging-user-password
+      email: example@gmail.com
+      server: harbor-core.jarvis.local
+#   clusterA:
+#     kubeconfig:
+#     harbor-ca:
+#     helm:
+#       username:
+#       password:
+#     helm-staging:
+#       username:
+#       password:
+#     harbor:
+#       username:
+#       password:
+#       email:
+#       server:
+#     harbor-staging:
+#       username:
+#       password:
+#       email:
+#       server:
+# sil:
+#   clusterA:
+#     kubeconfig:
+#     harbor-ca:
+#     helm:
+#       username:
+#       password:
+#     helm-staging:
+#       username:
+#       password:
+#     harbor:
+#       username:
+#       password:
+#       email:
+#       server:
+#     harbor-staging:
+#       username:
+#       password:
+#       email:
+#       server:
+#   clusterB:
+#     kubeconfig:
+#     harbor-ca:
+#     helm:
+#       username:
+#       password:
+#     helm-staging:
+#       username:
+#       password:
+#     harbor:
+#       username:
+#       password:
+#       email:
+#       server:
+#     harbor-staging:
+#       username:
+#       password:
+#       email:
+#       server:
+# prod:
+#   clusterA:
+#     kubeconfig:
+#     harbor-ca:
+#     helm:
+#       username:
+#       password:
+#     helm-staging:
+#       username:
+#       password:
+#     harbor:
+#       username:
+#       password:
+#       email:
+#       server:
+#     harbor-staging:
+#       username:
+#       password:
+#       email:
+#       server:
+#   clusterB:
+#     kubeconfig:
+#     harbor-ca:
+#     helm:
+#       username:
+#       password:
+#     helm-staging:
+#       username:
+#       password:
+#     harbor:
+#       username:
+#       password:
+#       email:
+#       server:
+#     harbor-staging:
+#       username:
+#       password:
+#       email:
+#       server:

tools/gate/jarvis/5G-SA-core/mongodb/jarvis/development-pipeline/templates/config_map.yaml

@@ -69,6 +69,8 @@ data:
       "chart_repository": "jarvis-harbor",
       "docker_registry": "harbor-core.jarvis.local",
       "harbor_secret_mounted_path": "/workspace/helm-creds",
+      "harbor_secret_pull_mounted_path": "/workspace/helm-creds/pull",
+      "harbor_secret_push_mounted_path": "/workspace/helm-creds/push",
       "clone_async_batch_size": 20,
       "chart_async_batch_size": 20,
       "image_async_batch_size": 2

tools/gate/jarvis/5G-SA-core/nrf/jarvis.yaml

@@ -0,0 +1,121 @@
+dev:
+  jarvis-aio:
+    # Left blank as the kubeconfig is added in during execution of the 800-deploy-jarvis-projects.sh script
+    # The kubeconfig is added in from ~/.kube/config
+    kubeconfig:
+    # Left blank as the harbor-ca is added in during execution of the 800-deploy-jarvis-projects.sh script
+    # The harbor-ca is added in from /etc/jarvis/certs/ca/ca.pem
+    harbor-ca:
+    helm:
+      username: nrf-harbor
+      password: harbor-user-password
+    helm-staging:
+      username: nrf-harbor-staging
+      password: harbor-staging-user-password
+    harbor:
+      username: nrf-harbor
+      password: harbor-user-password
+      email: example@gmail.com
+      server: harbor-core.jarvis.local
+    harbor-staging:
+      username: nrf-harbor-staging
+      password: harbor-staging-user-password
+      email: example@gmail.com
+      server: harbor-core.jarvis.local
+#   clusterA:
+#     kubeconfig:
+#     harbor-ca:
+#     helm:
+#       username:
+#       password:
+#     helm-staging:
+#       username:
+#       password:
+#     harbor:
+#       username:
+#       password:
+#       email:
+#       server:
+#     harbor-staging:
+#       username:
+#       password:
+#       email:
+#       server:
+# sil:
+#   clusterA:
+#     kubeconfig:
+#     harbor-ca:
+#     helm:
+#       username:
+#       password:
+#     helm-staging:
+#       username:
+#       password:
+#     harbor:
+#       username:
+#       password:
+#       email:
+#       server:
+#     harbor-staging:
+#       username:
+#       password:
+#       email:
+#       server:
+#   clusterB:
+#     kubeconfig:
+#     harbor-ca:
+#     helm:
+#       username:
+#       password:
+#     helm-staging:
+#       username:
+#       password:
+#     harbor:
+#       username:
+#       password:
+#       email:
+#       server:
+#     harbor-staging:
+#       username:
+#       password:
+#       email:
+#       server:
+# prod:
+#   clusterA:
+#     kubeconfig:
+#     harbor-ca:
+#     helm:
+#       username:
+#       password:
+#     helm-staging:
+#       username:
+#       password:
+#     harbor:
+#       username:
+#       password:
+#       email:
+#       server:
+#     harbor-staging:
+#       username:
+#       password:
+#       email:
+#       server:
+#   clusterB:
+#     kubeconfig:
+#     harbor-ca:
+#     helm:
+#       username:
+#       password:
+#     helm-staging:
+#       username:
+#       password:
+#     harbor:
+#       username:
+#       password:
+#       email:
+#       server:
+#     harbor-staging:
+#       username:
+#       password:
+#       email:
+#       server:

tools/gate/jarvis/5G-SA-core/nrf/jarvis/development-pipeline/templates/config_map.yaml

@@ -73,6 +73,8 @@ data:
       "chart_repository": "jarvis-harbor",
       "docker_registry": "harbor-core.jarvis.local",
       "harbor_secret_mounted_path": "/workspace/helm-creds",
+      "harbor_secret_pull_mounted_path": "/workspace/helm-creds/pull",
+      "harbor_secret_push_mounted_path": "/workspace/helm-creds/push",
       "clone_async_batch_size": 20,
       "chart_async_batch_size": 20,
       "image_async_batch_size": 2

tools/gate/jarvis/5G-SA-core/pcf/jarvis.yaml

@@ -0,0 +1,101 @@
+dev:
+  jarvis-aio:
+    # Left blank as the kubeconfig is added in during execution of the 800-deploy-jarvis-projects.sh script
+    # The kubeconfig is added in from ~/.kube/config
+    kubeconfig:
+    # Left blank as the harbor-ca is added in during execution of the 800-deploy-jarvis-projects.sh script
+    # The harbor-ca is added in from /etc/jarvis/certs/ca/ca.pem
+    harbor-ca:
+    helm:
+      username: pcf-harbor
+      password: harbor-user-password
+    helm-staging:
+      username: pcf-harbor-staging
+      password: harbor-staging-user-password
+    harbor:
+      username: pcf-harbor
+      password: harbor-user-password
+      email: example@gmail.com
+      server: harbor-core.jarvis.local
+    harbor-staging:
+      username: pcf-harbor-staging
+      password: harbor-staging-user-password
+      email: example@gmail.com
+      server: harbor-core.jarvis.local
+#   clusterA:
+#     kubeconfig:
+#     harbor-ca:
+#     helm:
+#       username:
+#       password:
+#     helm-staging:
+#       username:
+#       password:
+#     harbor:
+#       username:
+#       password:
+#     harbor-staging:
+#       username:
+#       password:
+# sil:
+#   clusterA:
+#     kubeconfig:
+#     harbor-ca:
+#     helm:
+#       username:
+#       password:
+#     helm-staging:
+#       username:
+#       password:
+#     harbor:
+#       username:
+#       password:
+#     harbor-staging:
+#       username:
+#       password:
+#   clusterB:
+#     kubeconfig:
+#     harbor-ca:
+#     helm:
+#       username:
+#       password:
+#     helm-staging:
+#       username:
+#       password:
+#     harbor:
+#       username:
+#       password:
+#     harbor-staging:
+#       username:
+#       password:
+# prod:
+#   clusterA:
+#     kubeconfig:
+#     harbor-ca:
+#     helm:
+#       username:
+#       password:
+#     helm-staging:
+#       username:
+#       password:
+#     harbor:
+#       username:
+#       password:
+#     harbor-staging:
+#       username:
+#       password:
+#   clusterB:
+#     kubeconfig:
+#     harbor-ca:
+#     helm:
+#       username:
+#       password:
+#     helm-staging:
+#       username:
+#       password:
+#     harbor:
+#       username:
+#       password:
+#     harbor-staging:
+#       username:
+#       password:

tools/gate/jarvis/5G-SA-core/pcf/jarvis/development-pipeline/templates/config_map.yaml

@@ -73,6 +73,8 @@ data:
       "chart_repository": "jarvis-harbor",
       "docker_registry": "harbor-core.jarvis.local",
       "harbor_secret_mounted_path": "/workspace/helm-creds",
+      "harbor_secret_pull_mounted_path": "/workspace/helm-creds/pull",
+      "harbor_secret_push_mounted_path": "/workspace/helm-creds/push",
       "clone_async_batch_size": 20,
       "chart_async_batch_size": 20,
       "image_async_batch_size": 2

tools/gate/jarvis/5G-SA-core/smf/jarvis.yaml

@@ -0,0 +1,121 @@
+dev:
+  jarvis-aio:
+    # Left blank as the kubeconfig is added in during execution of the 800-deploy-jarvis-projects.sh script
+    # The kubeconfig is added in from ~/.kube/config
+    kubeconfig:
+    # Left blank as the harbor-ca is added in during execution of the 800-deploy-jarvis-projects.sh script
+    # The harbor-ca is added in from /etc/jarvis/certs/ca/ca.pem
+    harbor-ca:
+    helm:
+      username: smf-harbor
+      password: harbor-user-password
+    helm-staging:
+      username: smf-harbor-staging
+      password: harbor-staging-user-password
+    harbor:
+      username: smf-harbor
+      password: harbor-user-password
+      email: example@gmail.com
+      server: harbor-core.jarvis.local
+    harbor-staging:
+      username: smf-harbor-staging
+      password: harbor-staging-user-password
+      email: example@gmail.com
+      server: harbor-core.jarvis.local
+#   clusterA:
+#     kubeconfig:
+#     harbor-ca:
+#     helm:
+#       username:
+#       password:
+#     helm-staging:
+#       username:
+#       password:
+#     harbor:
+#       username:
+#       password:
+#       email:
+#       server:
+#     harbor-staging:
+#       username:
+#       password:
+#       email:
+#       server:
+# sil:
+#   clusterA:
+#     kubeconfig:
+#     harbor-ca:
+#     helm:
+#       username:
+#       password:
+#     helm-staging:
+#       username:
+#       password:
+#     harbor:
+#       username:
+#       password:
+#       email:
+#       server:
+#     harbor-staging:
+#       username:
+#       password:
+#       email:
+#       server:
+#   clusterB:
+#     kubeconfig:
+#     harbor-ca:
+#     helm:
+#       username:
+#       password:
+#     helm-staging:
+#       username:
+#       password:
+#     harbor:
+#       username:
+#       password:
+#       email:
+#       server:
+#     harbor-staging:
+#       username:
+#       password:
+#       email:
+#       server:
+# prod:
+#   clusterA:
+#     kubeconfig:
+#     harbor-ca:
+#     helm:
+#       username:
+#       password:
+#     helm-staging:
+#       username:
+#       password:
+#     harbor:
+#       username:
+#       password:
+#       email:
+#       server:
+#     harbor-staging:
+#       username:
+#       password:
+#       email:
+#       server:
+#   clusterB:
+#     kubeconfig:
+#     harbor-ca:
+#     helm:
+#       username:
+#       password:
+#     helm-staging:
+#       username:
+#       password:
+#     harbor:
+#       username:
+#       password:
+#       email:
+#       server:
+#     harbor-staging:
+#       username:
+#       password:
+#       email:
+#       server:

tools/gate/jarvis/5G-SA-core/smf/jarvis/development-pipeline/templates/config_map.yaml

@@ -73,6 +73,8 @@ data:
       "chart_repository": "jarvis-harbor",
       "docker_registry": "harbor-core.jarvis.local",
       "harbor_secret_mounted_path": "/workspace/helm-creds",
+      "harbor_secret_pull_mounted_path": "/workspace/helm-creds/pull",
+      "harbor_secret_push_mounted_path": "/workspace/helm-creds/push",
       "clone_async_batch_size": 20,
       "chart_async_batch_size": 20,
       "image_async_batch_size": 2

tools/gate/jarvis/5G-SA-core/udm/jarvis.yaml

@@ -0,0 +1,121 @@
+dev:
+  jarvis-aio:
+    # Left blank as the kubeconfig is added in during execution of the 800-deploy-jarvis-projects.sh script
+    # The kubeconfig is added in from ~/.kube/config
+    kubeconfig:
+    # Left blank as the harbor-ca is added in during execution of the 800-deploy-jarvis-projects.sh script
+    # The harbor-ca is added in from /etc/jarvis/certs/ca/ca.pem
+    harbor-ca:
+    helm:
+      username: udm-harbor
+      password: harbor-user-password
+    helm-staging:
+      username: udm-harbor-staging
+      password: harbor-staging-user-password
+    harbor:
+      username: udm-harbor
+      password: harbor-user-password
+      email: example@gmail.com
+      server: harbor-core.jarvis.local
+    harbor-staging:
+      username: udm-harbor-staging
+      password: harbor-staging-user-password
+      email: example@gmail.com
+      server: harbor-core.jarvis.local
+#   clusterA:
+#     kubeconfig:
+#     harbor-ca:
+#     helm:
+#       username:
+#       password:
+#     helm-staging:
+#       username:
+#       password:
+#     harbor:
+#       username:
+#       password:
+#       email:
+#       server:
+#     harbor-staging:
+#       username:
+#       password:
+#       email:
+#       server:
+# sil:
+#   clusterA:
+#     kubeconfig:
+#     harbor-ca:
+#     helm:
+#       username:
+#       password:
+#     helm-staging:
+#       username:
+#       password:
+#     harbor:
+#       username:
+#       password:
+#       email:
+#       server:
+#     harbor-staging:
+#       username:
+#       password:
+#       email:
+#       server:
+#   clusterB:
+#     kubeconfig:
+#     harbor-ca:
+#     helm:
+#       username:
+#       password:
+#     helm-staging:
+#       username:
+#       password:
+#     harbor:
+#       username:
+#       password:
+#       email:
+#       server:
+#     harbor-staging:
+#       username:
+#       password:
+#       email:
+#       server:
+# prod:
+#   clusterA:
+#     kubeconfig:
+#     harbor-ca:
+#     helm:
+#       username:
+#       password:
+#     helm-staging:
+#       username:
+#       password:
+#     harbor:
+#       username:
+#       password:
+#       email:
+#       server:
+#     harbor-staging:
+#       username:
+#       password:
+#       email:
+#       server:
+#   clusterB:
+#     kubeconfig:
+#     harbor-ca:
+#     helm:
+#       username:
+#       password:
+#     helm-staging:
+#       username:
+#       password:
+#     harbor:
+#       username:
+#       password:
+#       email:
+#       server:
+#     harbor-staging:
+#       username:
+#       password:
+#       email:
+#       server:

tools/gate/jarvis/5G-SA-core/udm/jarvis/development-pipeline/templates/config_map.yaml

@@ -73,6 +73,8 @@ data:
       "chart_repository": "jarvis-harbor",
       "docker_registry": "harbor-core.jarvis.local",
       "harbor_secret_mounted_path": "/workspace/helm-creds",
+      "harbor_secret_pull_mounted_path": "/workspace/helm-creds/pull",
+      "harbor_secret_push_mounted_path": "/workspace/helm-creds/push",
       "clone_async_batch_size": 20,
       "chart_async_batch_size": 20,
       "image_async_batch_size": 2

tools/gate/jarvis/5G-SA-core/udr/jarvis.yaml

@@ -0,0 +1,121 @@
+dev:
+  jarvis-aio:
+    # Left blank as the kubeconfig is added in during execution of the 800-deploy-jarvis-projects.sh script
+    # The kubeconfig is added in from ~/.kube/config
+    kubeconfig:
+    # Left blank as the harbor-ca is added in during execution of the 800-deploy-jarvis-projects.sh script
+    # The harbor-ca is added in from /etc/jarvis/certs/ca/ca.pem
+    harbor-ca:
+    helm:
+      username: udr-harbor
+      password: harbor-user-password
+    helm-staging:
+      username: udr-harbor-staging
+      password: harbor-staging-user-password
+    harbor:
+      username: udr-harbor
+      password: harbor-user-password
+      email: example@gmail.com
+      server: harbor-core.jarvis.local
+    harbor-staging:
+      username: udr-harbor-staging
+      password: harbor-staging-user-password
+      email: example@gmail.com
+      server: harbor-core.jarvis.local
+#   clusterA:
+#     kubeconfig:
+#     harbor-ca:
+#     helm:
+#       username:
+#       password:
+#     helm-staging:
+#       username:
+#       password:
+#     harbor:
+#       username:
+#       password:
+#       email:
+#       server:
+#     harbor-staging:
+#       username:
+#       password:
+#       email:
+#       server:
+# sil:
+#   clusterA:
+#     kubeconfig:
+#     harbor-ca:
+#     helm:
+#       username:
+#       password:
+#     helm-staging:
+#       username:
+#       password:
+#     harbor:
+#       username:
+#       password:
+#       email:
+#       server:
+#     harbor-staging:
+#       username:
+#       password:
+#       email:
+#       server:
+#   clusterB:
+#     kubeconfig:
+#     harbor-ca:
+#     helm:
+#       username:
+#       password:
+#     helm-staging:
+#       username:
+#       password:
+#     harbor:
+#       username:
+#       password:
+#       email:
+#       server:
+#     harbor-staging:
+#       username:
+#       password:
+#       email:
+#       server:
+# prod:
+#   clusterA:
+#     kubeconfig:
+#     harbor-ca:
+#     helm:
+#       username:
+#       password:
+#     helm-staging:
+#       username:
+#       password:
+#     harbor:
+#       username:
+#       password:
+#       email:
+#       server:
+#     harbor-staging:
+#       username:
+#       password:
+#       email:
+#       server:
+#   clusterB:
+#     kubeconfig:
+#     harbor-ca:
+#     helm:
+#       username:
+#       password:
+#     helm-staging:
+#       username:
+#       password:
+#     harbor:
+#       username:
+#       password:
+#       email:
+#       server:
+#     harbor-staging:
+#       username:
+#       password:
+#       email:
+#       server:

tools/gate/jarvis/5G-SA-core/udr/jarvis/development-pipeline/templates/config_map.yaml

@@ -70,6 +70,8 @@ data:
       "chart_repository": "jarvis-harbor",
       "docker_registry": "harbor-core.jarvis.local",
       "harbor_secret_mounted_path": "/workspace/helm-creds",
+      "harbor_secret_pull_mounted_path": "/workspace/helm-creds/pull",
+      "harbor_secret_push_mounted_path": "/workspace/helm-creds/push",
       "clone_async_batch_size": 20,
       "chart_async_batch_size": 20,
       "image_async_batch_size": 2

tools/gate/jarvis/650-temporary-setup.sh

@@ -1,18 +1,8 @@
 #!/bin/bash
 set -ex
 
-ldap_username="jarvis"
+# development-pipeline namespace is needed by the mongodb bitnami helm release
-ldap_password="password"
-ldap_email="jarvis@cluster.local"
-harbor_core="harbor-core.jarvis.local" #Defined in harbor overrides, TODO, extract from there
-
-#TODO(staceyF) Put this into appropriate jarvis-system tasks
 kubectl create ns development-pipeline || true
-kubectl create secret generic harbor-ca --from-file=harbor-ca=/etc/jarvis/certs/ca/ca.pem -n development-pipeline || true
-kubectl create secret generic kubeconfig-secret --from-file=kubeconfig=$HOME/.kube/config -n development-pipeline || true
-#NOTE Will not be required once Harbor is backed by LDAP
-kubectl create secret generic harbor-basic-auth --from-literal=username=$ldap_username --from-literal=password=$ldap_password -n development-pipeline || true
-kubectl create secret docker-registry harbor-docker-auth --docker-username=$ldap_username --docker-password=$ldap_password --docker-email=$ldap_email --docker-server=$harbor_core -n development-pipeline || true
 
 cd ./tools/gate/jarvis/standard-container
-sudo docker build -t standard-container:1.0 .
+sudo docker build -t standard-container:1.0 .

tools/gate/jarvis/800-deploy-jarvis-projects.sh

@@ -62,6 +62,11 @@ EOF
   git clone ssh://${ldap_username}@gerrit.jarvis.local:29418/${jarvis_project}.git "${jarvis_sanity_repo}"
   pushd "${jarvis_sanity_repo}"
   popd
+  # Add kubeconfig and ca to jarvis.yaml as single line base64 encoded so that to preserve the indentation required to be a valid kubeconfig
+  KUBECONFIG=$(base64 -w 0 ~/.kube/config)
+  CRT=$(base64 -w0 /etc/jarvis/certs/ca/ca.pem)
+  echo "$KUBECONFIG" | xargs -n 1 -I {} yq eval -i '.dev."jarvis-aio".kubeconfig = "{}"' tools/gate/jarvis/5G-SA-core/${jarvis_project}/jarvis.yaml
+  echo "$CRT" | xargs -n 1 -I {} yq eval -i '.dev."jarvis-aio"."harbor-ca" = "{}"' tools/gate/jarvis/5G-SA-core/${jarvis_project}/jarvis.yaml
   #Copy CNF code, development-pipeline and standard-container into each CNF git repository
   cp -a tools/gate/jarvis/5G-SA-core/${jarvis_project}/. "${jarvis_sanity_repo}"
   cp -a tools/gate/jarvis/development-pipeline/* "${jarvis_sanity_repo}/jarvis/development-pipeline"

tools/gate/jarvis/development-pipeline/templates/task-promote.yaml

@@ -27,7 +27,10 @@ spec:
       - name: HELM_DATA_HOME
         value: /root/.local/share/helm
       volumeMounts:
-        - mountPath: /tekton/home/.docker/config.json
+        - mountPath: /tekton/home/.docker/pull/config.json
+          name: image-pull-creds
+          subPath: .dockerconfigjson
+        - mountPath: /tekton/home/.docker/push/config.json
           name: image-push-creds
           subPath: .dockerconfigjson
         - mountPath: /certs/client
@@ -35,8 +38,10 @@ spec:
         - mountPath: /usr/local/share/ca-certificates/harbor-ca.crt
           name: harbor-ca
           subPath: harbor-ca
-        - mountPath: /workspace/helm-creds
+        - mountPath: /workspace/helm-creds/push
-          name: helm-publish-creds
+          name: helm-push-creds
+        - mountPath: /workspace/helm-creds/pull
+          name: helm-pull-creds
       script: |
         /jarvis/promote_artifacts.sh
   sidecars:
@@ -66,12 +71,18 @@ spec:
   volumes:
     - name: dind-certs
       emptyDir: {}
-    - name: image-push-creds
+    - name: image-pull-creds
       secret:
         secretName: harbor-docker-auth
-    - name: helm-publish-creds
+    - name: image-push-creds
+      secret:
+        secretName: harbor-docker-auth-promote
+    - name: helm-pull-creds
       secret:
         secretName: harbor-basic-auth
+    - name: helm-push-creds
+      secret:
+        secretName: harbor-basic-auth-promote
     - name: harbor-ca
       secret:
         secretName: harbor-ca

tools/gate/jarvis/standard-container/Dockerfile

@@ -28,6 +28,7 @@ RUN apt-get update ;\
         docker-ce-cli ;\
     pip3 install --upgrade wheel ;\
     pip3 install ansible==2.10.7 ;\
+    pip3 install yq ;\
     ansible-galaxy collection install community.kubernetes ;\
     pip3 install docker ;\
     # Install kubectl

tools/gate/jarvis/standard-container/assets/playbooks/promote-microflow.yaml

@@ -11,12 +11,12 @@
 
     - name: Get list of internal staging repos
       when: ( stage == "promote_chart")
-      shell: "jq -c '.[] | {repo: (\"https://{{ chart_registry_url }}/\"+.project+\"-staging\"), secret: \"{{ harbor_secret_mounted_path }}\", name:  (.project+\"-staging\")}'  {{ loop_source }}  | sort | uniq"
+      shell: "jq -c '.[] | {repo: (\"https://{{ chart_registry_url }}/\"+.project+\"-staging\"), secret: \"{{ harbor_secret_pull_mounted_path }}\", name:  (.project+\"-staging\")}'  {{ loop_source }}  | sort | uniq"
       register: staging_internal_repos
 
     - name: Get list of internal stable repos
       when: ( stage == "promote_chart")
-      shell: "jq -c '.[] | {repo: (\"https://{{ chart_registry_url }}/\"+.project), secret: \"{{ harbor_secret_mounted_path }}\", name: .project}'  {{ loop_source }}  | sort | uniq"
+      shell: "jq -c '.[] | {repo: (\"https://{{ chart_registry_url }}/\"+.project), secret: \"{{ harbor_secret_push_mounted_path }}\", name: .project}'  {{ loop_source }}  | sort | uniq"
       register: internal_repos
 
     - include_tasks: ./roles/charts/tasks/setup_helm_repo.yaml

tools/gate/jarvis/standard-container/assets/playbooks/roles/promote/tasks/main.yaml

@@ -44,9 +44,7 @@
   when: ( stage == "promote_image")
   block:
     - name: Pull image from staging repo
-      docker_image:
+      shell: docker --config ~/.docker/pull pull "{{ docker_registry }}/{{ image.project }}-staging/{{ image.repo }}:{{ image.tag }}"
-        pull: true
-        name: "{{ docker_registry }}/{{ image.project }}-staging/{{ image.repo }}:{{ image.tag }}"
       loop: "{{ items }}"
       loop_control:
         loop_var: "image"
@@ -60,11 +58,9 @@
       include_tasks: ./roles/common/tasks/check_sync_status.yaml
 
     - name: Tag and push to promotion repository
-      docker_image:
+      shell: |
-        push: true
+        docker --config ~/.docker/push tag "{{ docker_registry }}/{{ image.project }}-staging/{{ image.repo }}:{{ image.tag }}" "{{ docker_registry }}/{{ image.project }}/{{ image.repo }}:{{ image.tag }}"
-        name: "{{ docker_registry }}/{{ image.project }}-staging/{{ image.repo }}"
+        docker --config ~/.docker/push push "{{ docker_registry }}/{{ image.project }}/{{ image.repo }}:{{ image.tag }}"
-        repository: "{{ docker_registry }}/{{ image.project }}/{{ image.repo }}"
-        tag: "{{ image.tag }}"
       loop: "{{ items }}"
       loop_control:
         loop_var: "image"