From 0750967a56c97e356633b78d01c16c7310924fd0 Mon Sep 17 00:00:00 2001
From: Victor Martinez <victormartinezrubio@gmail.com>
Date: Tue, 21 Jul 2020 12:51:59 +0100
Subject: [PATCH] [CI]: retry with sleep (#20065)

---
 .ci/scripts/report-codecov.sh | 18 +++++++++++++++++
 Jenkinsfile                   | 37 +++++++++++------------------------
 2 files changed, 29 insertions(+), 26 deletions(-)
 create mode 100755 .ci/scripts/report-codecov.sh

diff --git a/.ci/scripts/report-codecov.sh b/.ci/scripts/report-codecov.sh
new file mode 100755
index 00000000000..cd1ad38b73f
--- /dev/null
+++ b/.ci/scripts/report-codecov.sh
@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+set -exuo pipefail
+
+CODECOV_URL=https://codecov.io/bash
+if [ -e /usr/local/bin/bash_standard_lib.sh ] ; then
+    # shellcheck disable=SC1091
+    source /usr/local/bin/bash_standard_lib.sh
+    (retry 3 curl -sSLo codecov ${CODECOV_URL})
+else
+    curl -sSLo codecov ${CODECOV_URL}
+fi
+
+for i in "$@" ; do
+    FILE="${i}/build/coverage/full.cov"
+    if [ -f "${FILE}" ]; then
+        bash codecov -f "${FILE}"
+    fi
+done
diff --git a/Jenkinsfile b/Jenkinsfile
index 5b905d7d3f3..f666b0cf959 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -947,7 +947,9 @@ def withBeatsEnv(Map args = [:], Closure body) {
         if (archive) {
           archiveTestOutput(testResults: '**/build/TEST*.xml', artifacts: '**/build/TEST*.out')
         }
-        reportCoverage()
+        catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') {
+          sh(label: 'Report to Codecov', script: '.ci/scripts/report-codecov.sh auditbeat filebeat heartbeat journalbeat libbeat metricbeat packetbeat winlogbeat')
+        }
       }
     }
   }
@@ -1021,12 +1023,12 @@ def withBeatsEnvWin(Map args = [:], Closure body) {
 def installTools() {
   def i = 2 // Number of retries
   if(isUnix()) {
-    retry(i) { sh(label: "Install Go ${GO_VERSION}", script: ".ci/scripts/install-go.sh") }
-    retry(i) { sh(label: "Install docker-compose ${DOCKER_COMPOSE_VERSION}", script: ".ci/scripts/install-docker-compose.sh") }
-    retry(i) { sh(label: "Install Terraform ${TERRAFORM_VERSION}", script: ".ci/scripts/install-terraform.sh") }
-    retry(i) { sh(label: "Install Mage", script: "make mage") }
+    retryWithSleep(retries: i, seconds: 5, backoff: true){ sh(label: "Install Go ${GO_VERSION}", script: ".ci/scripts/install-go.sh") }
+    retryWithSleep(retries: i, seconds: 5, backoff: true){ sh(label: "Install docker-compose ${DOCKER_COMPOSE_VERSION}", script: ".ci/scripts/install-docker-compose.sh") }
+    retryWithSleep(retries: i, seconds: 5, backoff: true){ sh(label: "Install Terraform ${TERRAFORM_VERSION}", script: ".ci/scripts/install-terraform.sh") }
+    retryWithSleep(retries: i, seconds: 5, backoff: true){ sh(label: "Install Mage", script: "make mage") }
   } else {
-    retry(i) { bat(label: "Install Go/Mage/Python ${GO_VERSION}", script: ".ci/scripts/install-tools.bat") }
+    retryWithSleep(retries: i, seconds: 5, backoff: true){ bat(label: "Install Go/Mage/Python ${GO_VERSION}", script: ".ci/scripts/install-tools.bat") }
   }
 }
 
@@ -1129,23 +1131,6 @@ def k8sTest(versions){
   }
 }
 
-def reportCoverage(){
-  catchError(buildResult: 'SUCCESS', stageResult: 'UNSTABLE') {
-    retry(2){
-      sh(label: 'Report to Codecov', script: '''
-        curl -sSLo codecov https://codecov.io/bash
-        for i in auditbeat filebeat heartbeat libbeat metricbeat packetbeat winlogbeat journalbeat
-        do
-          FILE="${i}/build/coverage/full.cov"
-          if [ -f "${FILE}" ]; then
-            bash codecov -f "${FILE}"
-          fi
-        done
-      ''')
-    }
-  }
-}
-
 /**
 *  isChanged treats the patterns as regular expressions. In order to check if
 *  any file in a directoy is modified use `^<path to dir>/.*`.
@@ -1251,7 +1236,7 @@ def startCloudTestEnv(String name, environments = []) {
       try {
         for (environment in environments) {
           if (environment.cond || runAll) {
-            retry(2) {
+            retryWithSleep(retries: 2, seconds: 5, backoff: true){
               terraformApply(environment.dir)
             }
           }
@@ -1273,7 +1258,7 @@ def terraformCleanup(String stashName, String directory) {
     withCloudTestEnv() {
       withBeatsEnv(archive: false, withModule: false) {
         unstash("terraform-${stashName}")
-        retry(2) {
+        retryWithSleep(retries: 2, seconds: 5, backoff: true) {
           sh(label: "Terraform Cleanup", script: ".ci/scripts/terraform-cleanup.sh ${directory}")
         }
       }
@@ -1286,7 +1271,7 @@ def loadConfigEnvVars(){
   env.GO_VERSION = readFile(".go-version").trim()
 
   withEnv(["HOME=${env.WORKSPACE}"]) {
-    retry(2) { sh(label: "Install Go ${env.GO_VERSION}", script: ".ci/scripts/install-go.sh") }
+    retryWithSleep(retries: 2, seconds: 5, backoff: true){ sh(label: "Install Go ${env.GO_VERSION}", script: ".ci/scripts/install-go.sh") }
   }
 
   // Libbeat is the core framework of Beats. It has no additional dependencies