From 55aa34dde7dcfd7543af1db0c4a36ccd13b05711 Mon Sep 17 00:00:00 2001
From: Eric MacDonald <eric.macdonald@windriver.com>
Date: Wed, 5 Jun 2024 18:32:53 +0000
Subject: [PATCH] Add a ostree pull retry if the first pull attempt fails

This update adds 2 retries to the ostree pull operation for
system node or remote subcloud installs in attempt to handle
transient http server connection failure during the pull
operation.

Test Plan:

PASS: Verify successful system node install with retry code present.
PASS: Verify successful handling of a single transient connection loss
PASS: Verify handling of a persistent connection loss ; all tries fail
PASS: Verify successful subcloud install with retry code present.
PASS: Verify subcloud ostree pull retry handling ; 1,2 and max failures

Closes-Bug: 2068651
Change-Id: I538d5a179188966882c494731111d36b89c03415
Signed-off-by: Eric MacDonald <eric.macdonald@windriver.com>
---
 kickstart/files/kickstart.cfg | 22 ++++++++++++++----
 kickstart/files/miniboot.cfg  | 43 ++++++++++++++++++++++++++---------
 2 files changed, 49 insertions(+), 16 deletions(-)

diff --git a/kickstart/files/kickstart.cfg b/kickstart/files/kickstart.cfg
index 176e3861..c19df1a1 100644
--- a/kickstart/files/kickstart.cfg
+++ b/kickstart/files/kickstart.cfg
@@ -2042,11 +2042,23 @@ else
         fi
 
         ilog "Executing: ostree --repo=${repo} pull --depth=-1 --mirror ${instbr}:${instbr}"
-        ostree --repo=${repo} pull --depth=-1 --mirror ${instbr}:${instbr}
-        rc=$?
-        if [ $rc -ne 0 ]; then
-            report_failure_with_msg "ostree pull failed, rc=$rc"
-        fi
+        MAX_TRIES=3
+        RETRY_WAIT=10
+        for try in 1 2 3
+        do
+            ostree --repo=${repo} pull --depth=-1 --mirror ${instbr}:${instbr}
+            rc=$?
+            if [ ${rc} -ne 0 ]; then
+                if [ ${try} -lt ${MAX_TRIES} ] ; then
+                    wlog "ostree pull failed on try ${try} of ${MAX_TRIES}, rc=${rc} ; retry in ${RETRY_WAIT} seconds ..."
+                    sleep ${RETRY_WAIT}
+               else
+                    report_failure_with_msg "ostree pull failed, rc=${rc} ; max tries ${try} of ${MAX_TRIES}"
+                fi
+            else
+                break
+            fi
+        done
 
         umount ${PHYS_SYSROOT}
 
diff --git a/kickstart/files/miniboot.cfg b/kickstart/files/miniboot.cfg
index 5b76e846..4e6580ec 100644
--- a/kickstart/files/miniboot.cfg
+++ b/kickstart/files/miniboot.cfg
@@ -2178,11 +2178,23 @@ else
         fi
 
         ilog "Executing: ostree --repo=${repo} pull --depth=-1 --mirror ${instbr}:${instbr}"
-        ostree --repo=${repo} pull --depth=-1 --mirror ${instbr}:${instbr}
-        rc=$?
-        if [ $rc -ne 0 ]; then
-            report_failure_with_msg "ostree pull failed, rc=$rc"
-        fi
+        MAX_TRIES=3
+        RETRY_WAIT=10
+        for try in 1 2 3
+        do
+            ostree --repo=${repo} pull --depth=-1 --mirror ${instbr}:${instbr}
+            rc=$?
+            if [ ${rc} -ne 0 ]; then
+                if [ ${try} -lt ${MAX_TRIES} ] ; then
+                    wlog "ostree pull failed on try ${try} of ${MAX_TRIES}, rc=${rc} ; retry in ${RETRY_WAIT} seconds ..."
+                    sleep ${RETRY_WAIT}
+                else
+                    report_failure_with_msg "ostree pull failed, rc=${rc} ; max tries ${try} of ${MAX_TRIES}"
+                fi
+            else
+                break
+            fi
+        done
 
         if [ -n "${remote_insturl}" ]; then
             # In this case, we've initialized our ostree repo from local disk
@@ -2204,12 +2216,21 @@ else
                 ostree config --repo=${repo} set "remote \"${instbr}\"".gpg-verify false
             fi
             ilog "Executing ostree pull from ${remote_insturl}:"
-            ilog "ostree --repo=${repo} pull --depth=-1 --mirror ${instbr}:${instbr}"
-            ostree --repo=${repo} pull --depth=-1 --mirror ${instbr}:${instbr}
-            rc=$?
-            if [ $rc -ne 0 ]; then
-                report_failure_with_msg "ostree pull failed, rc=$rc"
-            fi
+            for try in 1 2 3
+            do
+                ostree --repo=${repo} pull --depth=-1 --mirror ${instbr}:${instbr}
+                rc=$?
+                if [ ${rc} -ne 0 ]; then
+                    if [ ${try} -lt ${MAX_TRIES} ] ; then
+                        wlog "ostree pull failed on try ${try} of ${MAX_TRIES}, rc=${rc} ; retry in ${RETRY_WAIT} seconds ..."
+                        sleep ${RETRY_WAIT}
+                    else
+                        report_failure_with_msg "ostree pull failed, rc=${rc} ; max tries ${try} of ${MAX_TRIES}"
+                    fi
+                else
+                    break
+                fi
+            done
         fi
         ilog "ostree log for ${repo}, branch=${instbr}:"
         ostree --repo=${repo} log ${instbr}