Hook up upgrade testing

Start running the new upgrade test right after building the QEMU image. In the AWS test job, run the upgrade test on AWS in parallel. For more information, see: coreos/mantle#1168
jlebon · Jan 28, 2020 · ab52e2a · ab52e2a
1 parent c3a57a9
commit ab52e2a
Show file tree

Hide file tree

Showing 3 changed files with 63 additions and 37 deletions.
diff --git a/Jenkinsfile b/Jenkinsfile
@@ -33,21 +33,6 @@ node {
     }
 }
 
-// Parse and handle the result of Kola
-boolean checkKolaSuccess(utils, currentBuild) {
-    // archive the image if the tests failed
-    def report = readJSON file: "tmp/kola/reports/report.json"
-    def result = report["result"]
-    print("kola result: ${result}")
-    if (result != "PASS") {
-        utils.shwrap("coreos-assembler compress --compressor xz")
-        archiveArtifacts "builds/latest/**/*.qcow2.xz"
-        currentBuild.result = 'FAILURE'
-        return false
-    }
-    return true
-}
-
 // Share with the Fedora testing account so we can test it afterwards
 FEDORA_AWS_TESTING_USER_ID = "013116697141"
 
@@ -259,6 +244,7 @@ podTemplate(cloud: 'openshift', label: pod_label, yaml: pod) {
             """)
         }
 
+        def meta_json
         def buildID = utils.shwrap_capture("readlink builds/latest")
         if (prevBuildID == buildID) {
             currentBuild.result = 'SUCCESS'
@@ -267,11 +253,11 @@ podTemplate(cloud: 'openshift', label: pod_label, yaml: pod) {
         } else {
             newBuildID = buildID
             currentBuild.description = "[${params.STREAM}] ⚡ ${newBuildID}"
+            meta_json = "builds/${newBuildID}/${basearch}/meta.json"
 
             // and insert the parent info into meta.json so we can display it in
             // the release browser and for sanity checking
             if (parent_commit && parent_version) {
-                def meta_json = "builds/${newBuildID}/${basearch}/meta.json"
                 def meta = readJSON file: meta_json
                 meta["fedora-coreos.parent-version"] = parent_version
                 meta["fedora-coreos.parent-commit"] = parent_commit
@@ -300,11 +286,11 @@ podTemplate(cloud: 'openshift', label: pod_label, yaml: pod) {
         stage('Kola:QEMU basic') {
             utils.shwrap("""
             cosa kola run --basic-qemu-scenarios --no-test-exit-error
-            tar -cf - tmp/kola/ | xz -c9 > kola-basic.tar.xz
+            tar -cf - tmp/kola/ | xz -c9 > kola-run-basic.tar.xz
             """)
-            archiveArtifacts "kola-basic.tar.xz"
+            archiveArtifacts "kola-run-basic.tar.xz"
         }
-        if (!checkKolaSuccess(utils, currentBuild)) {
+        if (!utils.checkKolaSuccess("tmp/kola", currentBuild)) {
             return
         }
 
@@ -313,11 +299,22 @@ podTemplate(cloud: 'openshift', label: pod_label, yaml: pod) {
             def parallel = ((cosa_memory_request_mb - 512) / 1024) as Integer
             utils.shwrap("""
             coreos-assembler kola run --parallel ${parallel} --no-test-exit-error
-            tar -cf - tmp/kola/ | xz -c9 > _kola_temp.tar.xz
+            tar -cf - tmp/kola/ | xz -c9 > kola-run.tar.xz
+            """)
+            archiveArtifacts "kola-run.tar.xz"
+        }
+        if (!utils.checkKolaSuccess("tmp/kola", currentBuild)) {
+            return
+        }
+
+        stage('Kola:QEMU upgrade') {
+            utils.shwrap("""
+            coreos-assembler kola --upgrades --no-test-exit-error
+            tar -cf - tmp/kola/ | xz -c9 > kola-run-upgrade.tar.xz
             """)
-            archiveArtifacts "_kola_temp.tar.xz"
+            archiveArtifacts "kola-run-upgrade.tar.xz"
         }
-        if (!checkKolaSuccess(utils, currentBuild)) {
+        if (!utils.checkKolaSuccess("tmp/kola", currentBuild)) {
             return
         }
 

diff --git a/Jenkinsfile.kola.aws b/Jenkinsfile.kola.aws
@@ -40,6 +40,7 @@ podTemplate(cloud: 'openshift', label: pod_label, yaml: pod) {
         def ami, ami_region
         def no_ami = false
 
+        def meta_json
         stage('Fetch Metadata') {
             utils.shwrap("""
             export AWS_CONFIG_FILE=\${AWS_FCOS_BUILDS_BOT_CONFIG}
@@ -48,7 +49,7 @@ podTemplate(cloud: 'openshift', label: pod_label, yaml: pod) {
             """)
 
             def basearch = utils.shwrap_capture("coreos-assembler basearch")
-            def meta_json = "builds/${params.VERSION}/${basearch}/meta.json"
+            meta_json = "builds/${params.VERSION}/${basearch}/meta.json"
             def meta = readJSON file: meta_json
             if (meta.amis.size() > 0) {
                 ami = meta['amis'][0]['hvm']
@@ -64,20 +65,33 @@ podTemplate(cloud: 'openshift', label: pod_label, yaml: pod) {
           return
         }
 
-        stage('AWS Kola Run') {
-          utils.shwrap("""
-          export AWS_CONFIG_FILE=\${AWS_FCOS_KOLA_BOT_CONFIG}
-          # use `cosa kola` here since it knows about blacklisted tests
-          cosa kola -- run -p aws --aws-ami ${ami} --aws-region ${ami_region} -b fcos -j 5 || :
-          tar -cf - tmp/kola | xz -c9 > kola_tmp.tar.xz
-          """)
-          archiveArtifacts "kola_tmp.tar.xz"
-        }
-
-        def report = readJSON file: "tmp/kola/reports/report.json"
-        if (report["result"] != "PASS") {
-          currentBuild.result = 'FAILURE'
-          return
+        stage('Kola') {
+            parallel aws: {
+                stage('Kola:AWS') {
+                  utils.shwrap("""
+                  export AWS_CONFIG_FILE=\${AWS_FCOS_KOLA_BOT_CONFIG}
+                  # use `cosa kola` here since it knows about blacklisted tests
+                  cosa kola run -p aws --aws-ami ${ami} --aws-region ${ami_region} -b fcos -j 5 --no-test-exit-error
+                  tar -cf - tmp/kola | xz -c9 > kola-run.tar.xz
+                  """)
+                  archiveArtifacts "kola-run.tar.xz"
+                }
+                if (!utils.checkKolaSuccess("tmp/kola", currentBuild)) {
+                    return
+                }
+            },
+            aws_upgrade: {
+                stage('Kola:AWS upgrade') {
+                    utils.shwrap("""
+                    coreos-assembler kola --upgrades -p=aws --aws-region ${ami_region} --no-test-exit-error
+                    tar -cf - tmp/kola/ | xz -c9 > kola-run-upgrade.tar.xz
+                    """)
+                    archiveArtifacts "kola-run-upgrade.tar.xz"
+                }
+                if (!utils.checkKolaSuccess("tmp/kola-upgrade", currentBuild)) {
+                    return
+                }
+            }
         }
     }}
 }
diff --git a/utils.groovy b/utils.groovy
@@ -43,6 +43,21 @@ def path_exists(path) {
     return shwrap_rc("test -e ${path}") == 0
 }
 
+// Parse and handle the result of Kola
+boolean checkKolaSuccess(dir, currentBuild) {
+    // archive the image if the tests failed
+    def report = readJSON file: "${dir}/reports/report.json"
+    def result = report["result"]
+    print("kola result: ${result}")
+    if (result != "PASS" && report["platform"] == "qemu-unpriv") {
+        shwrap("coreos-assembler compress --compressor xz")
+        archiveArtifacts "builds/latest/**/*.qcow2.xz"
+        currentBuild.result = 'FAILURE'
+        return false
+    }
+    return true
+}
+
 def aws_s3_cp_allow_noent(src, dest) {
     // see similar code in `cosa buildprep`
     shwrap("""