From d66cb1bf9155f2f6c49a31a175a66dfc915b55b2 Mon Sep 17 00:00:00 2001 From: FLETCHER Date: Fri, 26 Feb 2021 16:45:13 -0600 Subject: [PATCH] feat(cleanup) fix(order) fix(config) This ps allows for the cleanup of components to be controlled using the configmap for each sample cNF. This ps also changes the order of promote and functional stages to occur after merge. This ps also updates mongodb to use an upstream image and helm chart. Temporarily allows for anonymous access to images. Change deployment to use promoted image. Add remote_namespace to the config_map of mongodb to be deployed into external cluster. Change-Id: I70f095b6e54d1452dca93b2889d0d937b366a765 --- .../jarvis-project/templates/Job-project.yaml | 2 +- .../templates/config_map.yaml | 1 + .../templates/config_map.yaml | 3 +- .../templates/config_map.yaml | 4 ++- .../templates/config_map.yaml | 3 +- .../templates/config_map.yaml | 3 +- .../templates/config_map.yaml | 3 +- .../templates/config_map.yaml | 3 +- .../templates/config_map.yaml | 3 +- .../gate/jarvis/800-deploy-jarvis-projects.sh | 2 +- .../templates/pipeline.yaml | 22 +++++++------- .../assets/jarvis/cleanup.sh | 1 + .../roles/cleanup/tasks/cleanup.yaml | 30 +++++++++++-------- .../roles/functional/tasks/main.yaml | 8 ++--- .../assets/setup-cluster-config.sh | 5 ---- .../assets/test_helm_repo_basic_auth.yaml | 9 ------ 16 files changed, 53 insertions(+), 49 deletions(-) delete mode 100644 tools/gate/jarvis/standard-container/assets/setup-cluster-config.sh delete mode 100644 tools/gate/jarvis/standard-container/assets/test_helm_repo_basic_auth.yaml diff --git a/charts/jarvis-project/templates/Job-project.yaml b/charts/jarvis-project/templates/Job-project.yaml index bfcb5fa6..9e62b391 100644 --- a/charts/jarvis-project/templates/Job-project.yaml +++ b/charts/jarvis-project/templates/Job-project.yaml @@ -93,7 +93,7 @@ spec: --netrc-file /run/jarvis/harbor-netrc/harbor-netrc \ -H "accept: application/json" \ -H "Content-Type: application/json" \ - -d "{ \"project_name\": \"${PROJECT_NAME}\", \"public\": false, \"metadata\": { \"auto_scan\": \"true\" }}"` ;; + -d "{ \"project_name\": \"${PROJECT_NAME}\", \"public\": true, \"metadata\": { \"auto_scan\": \"true\" }}"` ;; *) echo "Unable to retrieve projects in harbor: ${PROJECT_CHECK_RESP} response code."; exit 1 esac done diff --git a/tools/gate/jarvis/5G-SA-core/amf/jarvis/development-pipeline/templates/config_map.yaml b/tools/gate/jarvis/5G-SA-core/amf/jarvis/development-pipeline/templates/config_map.yaml index e6add461..b456e0f1 100644 --- a/tools/gate/jarvis/5G-SA-core/amf/jarvis/development-pipeline/templates/config_map.yaml +++ b/tools/gate/jarvis/5G-SA-core/amf/jarvis/development-pipeline/templates/config_map.yaml @@ -92,5 +92,6 @@ data: } cleanup.json: | { + "environment_name": "development", "remove_artifacts": "true" } diff --git a/tools/gate/jarvis/5G-SA-core/ausf/jarvis/development-pipeline/templates/config_map.yaml b/tools/gate/jarvis/5G-SA-core/ausf/jarvis/development-pipeline/templates/config_map.yaml index 90a333f7..7f556416 100644 --- a/tools/gate/jarvis/5G-SA-core/ausf/jarvis/development-pipeline/templates/config_map.yaml +++ b/tools/gate/jarvis/5G-SA-core/ausf/jarvis/development-pipeline/templates/config_map.yaml @@ -92,5 +92,6 @@ data: } cleanup.json: | { - "remove_artifacts": "true" + "environment_name": "development", + "remove_artifacts": "true" } diff --git a/tools/gate/jarvis/5G-SA-core/mongodb/jarvis/development-pipeline/templates/config_map.yaml b/tools/gate/jarvis/5G-SA-core/mongodb/jarvis/development-pipeline/templates/config_map.yaml index fe4f173d..57251eaf 100644 --- a/tools/gate/jarvis/5G-SA-core/mongodb/jarvis/development-pipeline/templates/config_map.yaml +++ b/tools/gate/jarvis/5G-SA-core/mongodb/jarvis/development-pipeline/templates/config_map.yaml @@ -34,6 +34,7 @@ data: "version": "3.3.3", "build_from_source": false, "path": "", + "remote_namespace": "mongodb", "build": { "name": "https://charts.bitnami.com/bitnami", "local_repo": "bitnami_mongodb", @@ -84,5 +85,6 @@ data: } cleanup.json: | { - "remove_artifacts": "true" + "environment_name": "development", + "remove_artifacts": "false" } diff --git a/tools/gate/jarvis/5G-SA-core/nrf/jarvis/development-pipeline/templates/config_map.yaml b/tools/gate/jarvis/5G-SA-core/nrf/jarvis/development-pipeline/templates/config_map.yaml index 165dfcc4..2570eef5 100644 --- a/tools/gate/jarvis/5G-SA-core/nrf/jarvis/development-pipeline/templates/config_map.yaml +++ b/tools/gate/jarvis/5G-SA-core/nrf/jarvis/development-pipeline/templates/config_map.yaml @@ -92,5 +92,6 @@ data: } cleanup.json: | { - "remove_artifacts": "true" + "environment_name": "development", + "remove_artifacts": "true" } diff --git a/tools/gate/jarvis/5G-SA-core/pcf/jarvis/development-pipeline/templates/config_map.yaml b/tools/gate/jarvis/5G-SA-core/pcf/jarvis/development-pipeline/templates/config_map.yaml index 312f7079..0293e3cb 100644 --- a/tools/gate/jarvis/5G-SA-core/pcf/jarvis/development-pipeline/templates/config_map.yaml +++ b/tools/gate/jarvis/5G-SA-core/pcf/jarvis/development-pipeline/templates/config_map.yaml @@ -92,5 +92,6 @@ data: } cleanup.json: | { - "remove_artifacts": "true" + "environment_name": "development", + "remove_artifacts": "true" } diff --git a/tools/gate/jarvis/5G-SA-core/smf/jarvis/development-pipeline/templates/config_map.yaml b/tools/gate/jarvis/5G-SA-core/smf/jarvis/development-pipeline/templates/config_map.yaml index eb819b24..65748d5a 100644 --- a/tools/gate/jarvis/5G-SA-core/smf/jarvis/development-pipeline/templates/config_map.yaml +++ b/tools/gate/jarvis/5G-SA-core/smf/jarvis/development-pipeline/templates/config_map.yaml @@ -92,5 +92,6 @@ data: } cleanup.json: | { - "remove_artifacts": "true" + "environment_name": "development", + "remove_artifacts": "true" } diff --git a/tools/gate/jarvis/5G-SA-core/udm/jarvis/development-pipeline/templates/config_map.yaml b/tools/gate/jarvis/5G-SA-core/udm/jarvis/development-pipeline/templates/config_map.yaml index b0808f77..a504b918 100644 --- a/tools/gate/jarvis/5G-SA-core/udm/jarvis/development-pipeline/templates/config_map.yaml +++ b/tools/gate/jarvis/5G-SA-core/udm/jarvis/development-pipeline/templates/config_map.yaml @@ -92,5 +92,6 @@ data: } cleanup.json: | { - "remove_artifacts": "true" + "environment_name": "development", + "remove_artifacts": "true" } diff --git a/tools/gate/jarvis/5G-SA-core/udr/jarvis/development-pipeline/templates/config_map.yaml b/tools/gate/jarvis/5G-SA-core/udr/jarvis/development-pipeline/templates/config_map.yaml index 93b94d37..5120fb50 100644 --- a/tools/gate/jarvis/5G-SA-core/udr/jarvis/development-pipeline/templates/config_map.yaml +++ b/tools/gate/jarvis/5G-SA-core/udr/jarvis/development-pipeline/templates/config_map.yaml @@ -91,5 +91,6 @@ data: } cleanup.json: | { - "remove_artifacts": "true" + "environment_name": "development", + "remove_artifacts": "true" } diff --git a/tools/gate/jarvis/800-deploy-jarvis-projects.sh b/tools/gate/jarvis/800-deploy-jarvis-projects.sh index 484ac00b..8350f631 100755 --- a/tools/gate/jarvis/800-deploy-jarvis-projects.sh +++ b/tools/gate/jarvis/800-deploy-jarvis-projects.sh @@ -80,7 +80,7 @@ EOF git review change_id=$(git log -1 | grep Change-Id: | awk '{print $2}') popd - sleep 180 + sleep 60 if (( COUNTER == 0 )); then CHANGE_ID_COUNTER=$change_id diff --git a/tools/gate/jarvis/development-pipeline/templates/pipeline.yaml b/tools/gate/jarvis/development-pipeline/templates/pipeline.yaml index f7292db1..290cec12 100644 --- a/tools/gate/jarvis/development-pipeline/templates/pipeline.yaml +++ b/tools/gate/jarvis/development-pipeline/templates/pipeline.yaml @@ -95,7 +95,11 @@ spec: taskRef: name: deployment-manifests - - name: microflow-functional + - name: microflow-promote-artifacts + when: + - input: $(params.pipeline) + operator: in + values: ["merge"] runAfter: - microflow-deployment-manifests - microflow-k8s @@ -107,24 +111,22 @@ spec: - name: development_pipeline_data workspace: development_pipeline_data taskRef: - name: functional + name: promote - - name: microflow-promote-artifacts - when: - - input: $(params.pipeline) - operator: in - values: ["merge"] + - name: microflow-functional runAfter: - - microflow-functional + - microflow-promote-artifacts workspaces: - name: k8s_cluster_data workspace: k8s_cluster_data - name: development_pipeline_data workspace: development_pipeline_data taskRef: - name: promote - finally: + name: functional - name: microflow-cleanup + runAfter: + - microflow-functional + - microflow-promote-artifacts workspaces: - name: k8s_cluster_data workspace: k8s_cluster_data diff --git a/tools/gate/jarvis/standard-container/assets/jarvis/cleanup.sh b/tools/gate/jarvis/standard-container/assets/jarvis/cleanup.sh index 19a7f3a7..50822779 100755 --- a/tools/gate/jarvis/standard-container/assets/jarvis/cleanup.sh +++ b/tools/gate/jarvis/standard-container/assets/jarvis/cleanup.sh @@ -6,4 +6,5 @@ ansible-playbook -vvv "/playbooks/cleanup.yaml" -i hosts \ -e @"/workspace/development_pipeline_data/default.json" \ -e 'loop_chart_source="/workspace/development_pipeline_data/chart.json"' \ -e 'loop_image_source="/workspace/development_pipeline_data/image.json"' \ + -e @"/workspace/development_pipeline_data/cleanup.json" \ -e @"/workspace/development_pipeline_data/cluster.json" \ No newline at end of file diff --git a/tools/gate/jarvis/standard-container/assets/playbooks/roles/cleanup/tasks/cleanup.yaml b/tools/gate/jarvis/standard-container/assets/playbooks/roles/cleanup/tasks/cleanup.yaml index ef49bab2..eb05c1d8 100644 --- a/tools/gate/jarvis/standard-container/assets/playbooks/roles/cleanup/tasks/cleanup.yaml +++ b/tools/gate/jarvis/standard-container/assets/playbooks/roles/cleanup/tasks/cleanup.yaml @@ -2,17 +2,23 @@ - set_fact: _async: "{{ chart_async | default(600) }}" _delay: "{{ chart_check_status_delay | default(5) }}" + remove_artifacts: "{{ remove_artifacts }}" + environment_name: "{{ environment_name }}" -- name: Remove test release - shell: helm delete --kubeconfig="{{ cluster_kubeconfig_path }}/kubeconfig" "{{ chart.release_name }}" -n "{{ chart.namespace }}" - loop: "{{ charts }}" - loop_control: - loop_var: "chart" - async: "{{ _async }}" - poll: 0 - register: cleanup_async_results +- name: Cleanup Task + when: remove_artifacts == "true" and environment_name == "development" + block: + - name: Delete Helm Release + shell: helm delete --kubeconfig="{{ cluster_kubeconfig_path }}/kubeconfig" "{{ chart.release_name }}" -n "{{ chart.remote_namespace }}" + loop: "{{ charts }}" + loop_control: + loop_var: "chart" + async: "{{ _async }}" + poll: 0 + register: cleanup_async_results -- name: Wait for helm delete results - vars: - async_results: "{{ cleanup_async_results }}" - include_tasks: ./roles/common/tasks/check_sync_status.yaml + - name: Wait for helm delete results + vars: + async_results: "{{ cleanup_async_results }}" + include_tasks: ./roles/common/tasks/check_sync_status.yaml + become: true diff --git a/tools/gate/jarvis/standard-container/assets/playbooks/roles/functional/tasks/main.yaml b/tools/gate/jarvis/standard-container/assets/playbooks/roles/functional/tasks/main.yaml index ff00b7a0..98eb3782 100644 --- a/tools/gate/jarvis/standard-container/assets/playbooks/roles/functional/tasks/main.yaml +++ b/tools/gate/jarvis/standard-container/assets/playbooks/roles/functional/tasks/main.yaml @@ -3,7 +3,7 @@ _delay: "{{ chart_check_status_delay | default(5) }}" - name: Functional Deploy - when: ("{{ stage }}" == "deploy") + when: stage == "deploy" block: #Deploy CNF - name: Deploy charts @@ -30,10 +30,10 @@ cat {{ datapath }}/{{ chart.chart_name }}-overrides.json ; helm upgrade --install \ --kubeconfig="{{ cluster_kubeconfig_path }}/kubeconfig" \ - "{{ chart.release_name }}" "{{ chart.project }}-staging/{{ chart.chart_name }}" \ + "{{ chart.release_name }}" "{{ chart.project }}/{{ chart.chart_name }}" \ --version="{{ chart.version }}" \ --values="{{ datapath }}/{{ chart.chart_name }}-overrides.json" \ - --namespace="{{ chart.namespace }}" + --namespace="{{ chart.remote_namespace }}" args: executable: /bin/bash loop: "{{ charts }}" @@ -50,7 +50,7 @@ become: true - name: Functional Test - when: ( stage == "test") + when: stage == "test" block: #Test Deployed CNF - name: Chart currently has no Helm Tests, echo for now diff --git a/tools/gate/jarvis/standard-container/assets/setup-cluster-config.sh b/tools/gate/jarvis/standard-container/assets/setup-cluster-config.sh deleted file mode 100644 index 3c9d13f0..00000000 --- a/tools/gate/jarvis/standard-container/assets/setup-cluster-config.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/bash - -cp "$(workspaces.k8s_cluster_data.path)/default.json" "$(workspaces.development_pipeline_data.path)/default.json" -cp "$(workspaces.k8s_cluster_data.path)/cluster.json" "$(workspaces.development_pipeline_data.path)/cluster.json" -jq '.cluster_kubeconfig_path="$(workspaces.development_pipeline_data.path)/config"' "$(workspaces.development_pipeline_data.path)/cluster.json" > "$(workspaces.development_pipeline_data.path)/temp_cluster.json" && mv "$(workspaces.development_pipeline_data.path)/temp_cluster.json" "$(workspaces.development_pipeline_data.path)/cluster.json" diff --git a/tools/gate/jarvis/standard-container/assets/test_helm_repo_basic_auth.yaml b/tools/gate/jarvis/standard-container/assets/test_helm_repo_basic_auth.yaml deleted file mode 100644 index e0af040d..00000000 --- a/tools/gate/jarvis/standard-container/assets/test_helm_repo_basic_auth.yaml +++ /dev/null @@ -1,9 +0,0 @@ -apiVersion: v1 -kind: Secret -metadata: - name: harbor-helm-creds - namespace: development-pipeline -type: kubernetes.io/basic-auth -stringData: - username: admin - password: Harbor12345 \ No newline at end of file