From 8d5b34c5522c17f0246d208d0104c86e3325250e Mon Sep 17 00:00:00 2001
From: yatinkarel <ykarel@redhat.com>
Date: Tue, 2 Feb 2021 14:03:36 +0530
Subject: [PATCH] Add retries to module installation with r10k

Since the modules are git cloned when installed via r10k and
clones may fail due to some network glich, adding retries should
help in these cases.

We randomly hit failures during git clone in RDO promotion
jobs running on ci.centos.org, adding retries to r10k
installation to handle those random failures.

Added function 'retry_cmd' which can also be used
with other commands.

Change-Id: I5f3b94eaa5a65c76877363e9ca41684445c9cb12
---
 functions | 27 +++++++++++++++++++++++----
 1 file changed, 23 insertions(+), 4 deletions(-)

diff --git a/functions b/functions
index e6a9e250a..baee19e7e 100644
--- a/functions
+++ b/functions
@@ -3,15 +3,33 @@
 # functions - puppet-openstack-integration specific functions
 #
 
+# To retry a command until it succeed for a given
+# number of retries(default to 3).
+retry_cmd() {
+  local cmd=$1
+  local total_tries=${2:-3}
+  local delay=${3:-5}
+  local retry_count=1
+  local ret_code=1
+  until [[ ${ret_code} -eq 0 || ${retry_count} -gt ${total_tries} ]]; do
+      echo Retry count:-${retry_count}, Command:-$cmd
+      $cmd
+      ret_code=$?
+      ((retry_count++))
+      sleep ${delay}
+  done
+}
+
 # Install external Puppet modules with r10k
 # Uses the following variables:
 #
 # - ``SCRIPT_DIR`` must be set to script path
 # - ``GEM_BIN_DIR`` must be set to Gem bin directory
 install_external() {
-  r10k -v DEBUG puppetfile install \
+  install_cmd="r10k -v DEBUG puppetfile install \
     --puppetfile ${SCRIPT_DIR}/Puppetfile1 \
-    --moduledir ${PUPPETFILE_DIR}
+    --moduledir ${PUPPETFILE_DIR}"
+  retry_cmd "${install_cmd}"
 }
 
 # Install Puppet OpenStack modules from zuul checkouts
@@ -61,9 +79,10 @@ install_openstack() {
 install_all() {
   # When installing from local source, we want to install the current source
   # we're working from.
-  r10k -v DEBUG puppetfile install \
+  install_cmd="r10k -v DEBUG puppetfile install \
     --puppetfile ${SCRIPT_DIR}/Puppetfile \
-    --moduledir ${PUPPETFILE_DIR}
+    --moduledir ${PUPPETFILE_DIR}"
+  retry_cmd "${install_cmd}"
   cp -a ${SCRIPT_DIR} ${PUPPETFILE_DIR}/openstack_integration
 }