Add retries to module installation with r10k

Since the modules are git cloned when installed via r10k and
clones may fail due to some network glich, adding retries should
help in these cases.

We randomly hit failures during git clone in RDO promotion
jobs running on ci.centos.org, adding retries to r10k
installation to handle those random failures.

Added function 'retry_cmd' which can also be used
with other commands.

Change-Id: I5f3b94eaa5a65c76877363e9ca41684445c9cb12

functions

@@ -3,15 +3,33 @@
 # functions - puppet-openstack-integration specific functions
 #
 
+# To retry a command until it succeed for a given
+# number of retries(default to 3).
+retry_cmd() {
+  local cmd=$1
+  local total_tries=${2:-3}
+  local delay=${3:-5}
+  local retry_count=1
+  local ret_code=1
+  until [[ ${ret_code} -eq 0 || ${retry_count} -gt ${total_tries} ]]; do
+      echo Retry count:-${retry_count}, Command:-$cmd
+      $cmd
+      ret_code=$?
+      ((retry_count++))
+      sleep ${delay}
+  done
+}
+
 # Install external Puppet modules with r10k
 # Uses the following variables:
 #
 # - ``SCRIPT_DIR`` must be set to script path
 # - ``GEM_BIN_DIR`` must be set to Gem bin directory
 install_external() {
-  r10k -v DEBUG puppetfile install \
+  install_cmd="r10k -v DEBUG puppetfile install \
     --puppetfile ${SCRIPT_DIR}/Puppetfile1 \
-    --moduledir ${PUPPETFILE_DIR}
+    --moduledir ${PUPPETFILE_DIR}"
+  retry_cmd "${install_cmd}"
 }
 
 # Install Puppet OpenStack modules from zuul checkouts
@@ -61,9 +79,10 @@ install_openstack() {
 install_all() {
   # When installing from local source, we want to install the current source
   # we're working from.
-  r10k -v DEBUG puppetfile install \
+  install_cmd="r10k -v DEBUG puppetfile install \
     --puppetfile ${SCRIPT_DIR}/Puppetfile \
-    --moduledir ${PUPPETFILE_DIR}
+    --moduledir ${PUPPETFILE_DIR}"
+  retry_cmd "${install_cmd}"
   cp -a ${SCRIPT_DIR} ${PUPPETFILE_DIR}/openstack_integration
 }