Cleanup helm chart on demand during pre-upgrade

Change-Id: I80a5742d94be6e40f3215dfa7fc3258cca699acf
Signed-off-by: Ruslan Aliev <raliev@mirantis.com>

.zuul.yaml

@@ -40,7 +40,7 @@
     required-projects:
       - name: openstack/openstack-helm
       - name: openstack/openstack-helm-infra
-      - openstack/openstack-helm-plugin
+      - name: openstack/openstack-helm-plugin
       - name: airship/treasuremap
         override-checkout: v1.9
     irrelevant-files: &irrelevant-files
@@ -85,8 +85,10 @@
       helm_version: "v3.6.3"
       yq_version: "v4.6.0"
       crictl_version: "v1.26.1"
-      zuul_osh_infra_relative_path: ../openstack-helm-infra
-      gate_scripts_relative_path: ../openstack-helm-infra
+      zuul_osh_relative_path: ../../openstack/openstack-helm
+      zuul_osh_infra_relative_path: ../../openstack/openstack-helm-infra
+      zuul_treasuremap_relative_path: ../../airship/treasuremap
+      gate_scripts_relative_path: ../../openstack/openstack-helm-infra
       run_helm_tests: "no"
 
 - job:
@@ -97,10 +99,10 @@
     parent: armada-operator-base
     vars:
       site: airskiff
-      HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.13.2-linux-amd64.tar.gz
-      HTK_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
-      OSH_INFRA_COMMIT: cfff60ec10a6c386f38db79bb9f59a552c2b032f
-      OSH_COMMIT: 2d9457e34ca4200ed631466bd87569b0214c92e7
+      HELM_ARTIFACT_URL: https://get.helm.sh/helm-v3.14.2-linux-amd64.tar.gz
+      HTK_COMMIT: 05f2f45971abcf483189358d663e2b46c3fc2fe8
+      OSH_INFRA_COMMIT: 05f2f45971abcf483189358d663e2b46c3fc2fe8
+      OSH_COMMIT: 049e679939fbd3b0c659dd0977911b8dc3b5a015
       CLONE_ARMADA_OPERATOR: false
       DISTRO: ubuntu_focal
       DOCKER_REGISTRY: localhost:5000

api/v1/armadachart_types.go

@@ -104,7 +104,8 @@ type ArmadaChartUpgrade struct {
 }
 
 type ArmadaChartPreUpgrade struct {
-	Delete []ArmadaChartDeleteResource `json:"delete,omitempty"`
+	Cleanup bool                        `json:"cleanup,omitempty"`
+	Delete  []ArmadaChartDeleteResource `json:"delete,omitempty"`
 }
 
 // ArmadaChartDeleteResource defines the delete options of ArmadaChart

config/crd/bases/armada.airshipit.org_armadacharts.yaml

@@ -79,6 +79,8 @@ spec:
                 properties:
                   pre:
                     properties:
+                      cleanup:
+                        type: boolean
                       delete:
                         items:
                           description: ArmadaChartDeleteResource defines the delete

pkg/controller/armadachart_controller.go

@@ -75,7 +75,7 @@ func (r *ArmadaChartReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 	start := time.Now()
 	log := ctrl.LoggerFrom(ctx)
 
-	log.Info("reconciling has started")
+	log.Info("reconciliation started")
 
 	// Retrieve the custom resource
 	var ac armadav1.ArmadaChart
@@ -89,7 +89,7 @@ func (r *ArmadaChartReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 		controllerutil.AddFinalizer(&ac, armadav1.ArmadaChartFinalizer)
 		if err := r.Patch(ctx, &ac, patch); err != nil {
 			log.Error(err, "unable to register finalizer")
-			return ctrl.Result{}, err
+			return requeueRequired(ac, ctrl.Result{}, err)
 		}
 	}
 
@@ -103,7 +103,7 @@ func (r *ArmadaChartReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 	ac, result, err := r.reconcile(ctx, ac)
 	if updateStatusErr := r.patchStatus(ctx, &ac); updateStatusErr != nil {
 		log.Error(updateStatusErr, "unable to update status after reconciliation")
-		return ctrl.Result{Requeue: false}, updateStatusErr
+		return requeueRequired(ac, ctrl.Result{Requeue: false}, updateStatusErr)
 	}
 
 	// Log reconciliation duration
@@ -113,7 +113,7 @@ func (r *ArmadaChartReconciler) Reconcile(ctx context.Context, req ctrl.Request)
 	}
 	log.Info(durationMsg)
 
-	return result, err
+	return requeueRequired(ac, result, err)
 }
 
 func (r *ArmadaChartReconciler) reconcile(ctx context.Context, ac armadav1.ArmadaChart) (armadav1.ArmadaChart, ctrl.Result, error) {
@@ -231,9 +231,27 @@ func (r *ArmadaChartReconciler) reconcileChart(ctx context.Context,
 				}
 			}
 		}
+		if ac.Spec.Upgrade.PreUpgrade.Cleanup {
+			getter, err := r.buildRESTClientGetter(ctx, ac)
+			if err != nil {
+				return armadav1.ArmadaChartNotReady(ac, "DeleteHelmStatusFailed", err.Error()), err
+			}
+			run, err := runner.NewRunner(getter, ac.Spec.Namespace, ctrl.LoggerFrom(ctx))
+			if err != nil {
+				return armadav1.ArmadaChartNotReady(ac, "DeleteHelmStatusFailed", err.Error()), err
+			}
+			if err := run.Uninstall(ac); err != nil && !errors.Is(err, driver.ErrReleaseNotFound) {
+				return armadav1.ArmadaChartNotReady(ac, "DeleteHelmStatusFailed", err.Error()), err
+			}
+			log.Info("uninstalled Helm release for deleted resource")
 
-		log.Info("helm upgrade has started")
-		rel, err = run.Upgrade(ctx, ac, chrt, vals)
+			// Install action must be invoked
+			log.Info("helm install has started")
+			rel, err = run.Install(ctx, ac, chrt, vals)
+		} else {
+			log.Info("helm upgrade has started")
+			rel, err = run.Upgrade(ctx, ac, chrt, vals)
+		}
 	}
 
 	if err != nil {
@@ -462,3 +480,12 @@ func isUpdateRequired(ctx context.Context, release *release.Release, chrt *chart
 
 	return false
 }
+
+func requeueRequired(ac armadav1.ArmadaChart, res ctrl.Result, err error) (ctrl.Result, error) {
+	// We have to stop after 3 unsuccessful attempts on the same generation
+	if ac.Status.Failures > 3 {
+		return ctrl.Result{Requeue: false}, nil
+	}
+
+	return res, err
+}