feat(cleanup) fix(order) fix(config)

This ps allows for the cleanup of components to be controlled using
the configmap for each sample cNF.

This ps also changes the order of promote and functional stages to
occur after merge.

This ps also updates mongodb to use an upstream image and helm chart.

Temporarily allows for anonymous access to images.

Change deployment to use promoted image.

Add remote_namespace to the config_map of mongodb to be deployed into external cluster.

Change-Id: I70f095b6e54d1452dca93b2889d0d937b366a765

charts/jarvis-project/templates/Job-project.yaml

@@ -93,7 +93,7 @@ spec:
                   --netrc-file /run/jarvis/harbor-netrc/harbor-netrc \
                   -H "accept: application/json" \
                   -H "Content-Type: application/json" \
-                  -d "{ \"project_name\": \"${PROJECT_NAME}\", \"public\": false, \"metadata\": { \"auto_scan\": \"true\" }}"` ;;
+                  -d "{ \"project_name\": \"${PROJECT_NAME}\", \"public\": true, \"metadata\": { \"auto_scan\": \"true\" }}"` ;;
                   *) echo "Unable to retrieve projects in harbor: ${PROJECT_CHECK_RESP} response code."; exit 1
                   esac
                 done

tools/gate/jarvis/5G-SA-core/amf/jarvis/development-pipeline/templates/config_map.yaml

@@ -92,5 +92,6 @@ data:
     }
   cleanup.json: |
     {
+      "environment_name": "development",
       "remove_artifacts": "true"
     }

tools/gate/jarvis/5G-SA-core/ausf/jarvis/development-pipeline/templates/config_map.yaml

@@ -92,5 +92,6 @@ data:
     }
   cleanup.json: |
     {
-     "remove_artifacts": "true"
+      "environment_name": "development",
+      "remove_artifacts": "true"
     }

tools/gate/jarvis/5G-SA-core/mongodb/jarvis/development-pipeline/templates/config_map.yaml

@@ -34,6 +34,7 @@ data:
       "version": "3.3.3",
       "build_from_source": false,
       "path": "",
+      "remote_namespace": "mongodb",
       "build": {
         "name": "https://charts.bitnami.com/bitnami",
         "local_repo": "bitnami_mongodb",
@@ -84,5 +85,6 @@ data:
     }
   cleanup.json: |
     {
-     "remove_artifacts": "true"
+      "environment_name": "development",
+      "remove_artifacts": "false"
     }

tools/gate/jarvis/5G-SA-core/nrf/jarvis/development-pipeline/templates/config_map.yaml

@@ -92,5 +92,6 @@ data:
     }
   cleanup.json: |
     {
-     "remove_artifacts": "true"
+      "environment_name": "development",
+      "remove_artifacts": "true"
     }

tools/gate/jarvis/5G-SA-core/pcf/jarvis/development-pipeline/templates/config_map.yaml

@@ -92,5 +92,6 @@ data:
     }
   cleanup.json: |
     {
-     "remove_artifacts": "true"
+      "environment_name": "development",
+      "remove_artifacts": "true"
     }

tools/gate/jarvis/5G-SA-core/smf/jarvis/development-pipeline/templates/config_map.yaml

@@ -92,5 +92,6 @@ data:
     }
   cleanup.json: |
     {
-     "remove_artifacts": "true"
+      "environment_name": "development",
+      "remove_artifacts": "true"
     }

tools/gate/jarvis/5G-SA-core/udm/jarvis/development-pipeline/templates/config_map.yaml

@@ -92,5 +92,6 @@ data:
     }
   cleanup.json: |
     {
-     "remove_artifacts": "true"
+      "environment_name": "development",
+      "remove_artifacts": "true"
     }

tools/gate/jarvis/5G-SA-core/udr/jarvis/development-pipeline/templates/config_map.yaml

@@ -91,5 +91,6 @@ data:
     }
   cleanup.json: |
     {
-     "remove_artifacts": "true"
+      "environment_name": "development",
+      "remove_artifacts": "true"
     }

tools/gate/jarvis/800-deploy-jarvis-projects.sh

@@ -80,7 +80,7 @@ EOF
   git review
   change_id=$(git log -1 | grep Change-Id: | awk '{print $2}')
   popd
-  sleep 180
+  sleep 60
   if (( COUNTER == 0 ));
   then
     CHANGE_ID_COUNTER=$change_id

tools/gate/jarvis/development-pipeline/templates/pipeline.yaml

@@ -95,7 +95,11 @@ spec:
       taskRef:
         name: deployment-manifests
 
-    - name: microflow-functional
+    - name: microflow-promote-artifacts
+      when:
+        - input: $(params.pipeline)
+          operator: in
+          values: ["merge"]
       runAfter:
         - microflow-deployment-manifests
         - microflow-k8s
@@ -107,24 +111,22 @@ spec:
         - name: development_pipeline_data
           workspace: development_pipeline_data
       taskRef:
-        name: functional
+        name: promote
 
-    - name: microflow-promote-artifacts
+    - name: microflow-functional
-      when:
-        - input: $(params.pipeline)
-          operator: in
-          values: ["merge"]
       runAfter:
-        - microflow-functional
+        - microflow-promote-artifacts
       workspaces:
         - name: k8s_cluster_data
           workspace: k8s_cluster_data
         - name: development_pipeline_data
           workspace: development_pipeline_data
       taskRef:
-        name: promote
+        name: functional
-  finally:
     - name: microflow-cleanup
+      runAfter:
+        - microflow-functional
+        - microflow-promote-artifacts
       workspaces:
         - name: k8s_cluster_data
           workspace: k8s_cluster_data

tools/gate/jarvis/standard-container/assets/jarvis/cleanup.sh

@@ -6,4 +6,5 @@ ansible-playbook -vvv "/playbooks/cleanup.yaml" -i hosts \
   -e @"/workspace/development_pipeline_data/default.json" \
   -e 'loop_chart_source="/workspace/development_pipeline_data/chart.json"' \
   -e 'loop_image_source="/workspace/development_pipeline_data/image.json"' \
+  -e @"/workspace/development_pipeline_data/cleanup.json" \
   -e @"/workspace/development_pipeline_data/cluster.json"

tools/gate/jarvis/standard-container/assets/playbooks/roles/cleanup/tasks/cleanup.yaml

@@ -2,17 +2,23 @@
 - set_fact:
     _async: "{{ chart_async | default(600) }}"
     _delay: "{{ chart_check_status_delay | default(5) }}"
+    remove_artifacts: "{{ remove_artifacts }}"
+    environment_name: "{{ environment_name }}"
 
-- name: Remove test release
+- name: Cleanup Task
-  shell: helm delete --kubeconfig="{{ cluster_kubeconfig_path }}/kubeconfig" "{{ chart.release_name }}" -n "{{ chart.namespace }}"
+  when: remove_artifacts == "true" and environment_name == "development"
-  loop: "{{ charts }}"
+  block:
-  loop_control:
+    - name: Delete Helm Release
-    loop_var: "chart"
+      shell: helm delete --kubeconfig="{{ cluster_kubeconfig_path }}/kubeconfig" "{{ chart.release_name }}" -n "{{ chart.remote_namespace }}"
-  async: "{{ _async }}"
+      loop: "{{ charts }}"
-  poll: 0
+      loop_control:
-  register: cleanup_async_results
+        loop_var: "chart"
+      async: "{{ _async }}"
+      poll: 0
+      register: cleanup_async_results
 
-- name: Wait for helm delete results
+    - name: Wait for helm delete results
-  vars:
+      vars:
-    async_results: "{{ cleanup_async_results }}"
+        async_results: "{{ cleanup_async_results }}"
-  include_tasks: ./roles/common/tasks/check_sync_status.yaml
+      include_tasks: ./roles/common/tasks/check_sync_status.yaml
+  become: true

tools/gate/jarvis/standard-container/assets/playbooks/roles/functional/tasks/main.yaml

@@ -3,7 +3,7 @@
     _delay: "{{ chart_check_status_delay | default(5) }}"
 
 - name: Functional Deploy
-  when: ("{{ stage }}" == "deploy")
+  when: stage == "deploy"
   block:
     #Deploy CNF
     - name: Deploy charts
@@ -30,10 +30,10 @@
         cat {{ datapath }}/{{ chart.chart_name }}-overrides.json ;
         helm upgrade --install \
           --kubeconfig="{{ cluster_kubeconfig_path }}/kubeconfig" \
-          "{{ chart.release_name }}" "{{ chart.project }}-staging/{{ chart.chart_name }}" \
+          "{{ chart.release_name }}" "{{ chart.project }}/{{ chart.chart_name }}" \
           --version="{{ chart.version }}" \
           --values="{{ datapath }}/{{ chart.chart_name }}-overrides.json" \
-          --namespace="{{ chart.namespace }}"
+          --namespace="{{ chart.remote_namespace }}"
       args:
         executable: /bin/bash
       loop: "{{ charts }}"
@@ -50,7 +50,7 @@
   become: true
 
 - name: Functional Test
-  when: ( stage == "test")
+  when: stage == "test"
   block:
     #Test Deployed CNF
     - name: Chart currently has no Helm Tests, echo for now

tools/gate/jarvis/standard-container/assets/setup-cluster-config.sh

@@ -1,5 +0,0 @@
-#!/bin/bash
-
-cp "$(workspaces.k8s_cluster_data.path)/default.json" "$(workspaces.development_pipeline_data.path)/default.json"
-cp "$(workspaces.k8s_cluster_data.path)/cluster.json" "$(workspaces.development_pipeline_data.path)/cluster.json"
-jq '.cluster_kubeconfig_path="$(workspaces.development_pipeline_data.path)/config"' "$(workspaces.development_pipeline_data.path)/cluster.json" > "$(workspaces.development_pipeline_data.path)/temp_cluster.json" && mv "$(workspaces.development_pipeline_data.path)/temp_cluster.json" "$(workspaces.development_pipeline_data.path)/cluster.json"

tools/gate/jarvis/standard-container/assets/test_helm_repo_basic_auth.yaml

@@ -1,9 +0,0 @@
-apiVersion: v1
-kind: Secret
-metadata:
-  name: harbor-helm-creds
-  namespace: development-pipeline
-type: kubernetes.io/basic-auth
-stringData:
-  username: admin
-  password: Harbor12345