From ab52e2a4f8121981b5c4709634207da993fd5d8c Mon Sep 17 00:00:00 2001 From: Jonathan Lebon Date: Thu, 23 Jan 2020 17:06:30 -0500 Subject: [PATCH] Hook up upgrade testing Start running the new upgrade test right after building the QEMU image. In the AWS test job, run the upgrade test on AWS in parallel. For more information, see: https://github.com/coreos/mantle/pull/1168 --- Jenkinsfile | 41 +++++++++++++++++++---------------------- Jenkinsfile.kola.aws | 44 +++++++++++++++++++++++++++++--------------- utils.groovy | 15 +++++++++++++++ 3 files changed, 63 insertions(+), 37 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 2b8bdbd50..428250838 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -33,21 +33,6 @@ node { } } -// Parse and handle the result of Kola -boolean checkKolaSuccess(utils, currentBuild) { - // archive the image if the tests failed - def report = readJSON file: "tmp/kola/reports/report.json" - def result = report["result"] - print("kola result: ${result}") - if (result != "PASS") { - utils.shwrap("coreos-assembler compress --compressor xz") - archiveArtifacts "builds/latest/**/*.qcow2.xz" - currentBuild.result = 'FAILURE' - return false - } - return true -} - // Share with the Fedora testing account so we can test it afterwards FEDORA_AWS_TESTING_USER_ID = "013116697141" @@ -259,6 +244,7 @@ podTemplate(cloud: 'openshift', label: pod_label, yaml: pod) { """) } + def meta_json def buildID = utils.shwrap_capture("readlink builds/latest") if (prevBuildID == buildID) { currentBuild.result = 'SUCCESS' @@ -267,11 +253,11 @@ podTemplate(cloud: 'openshift', label: pod_label, yaml: pod) { } else { newBuildID = buildID currentBuild.description = "[${params.STREAM}] ⚡ ${newBuildID}" + meta_json = "builds/${newBuildID}/${basearch}/meta.json" // and insert the parent info into meta.json so we can display it in // the release browser and for sanity checking if (parent_commit && parent_version) { - def meta_json = "builds/${newBuildID}/${basearch}/meta.json" def meta = readJSON file: meta_json meta["fedora-coreos.parent-version"] = parent_version meta["fedora-coreos.parent-commit"] = parent_commit @@ -300,11 +286,11 @@ podTemplate(cloud: 'openshift', label: pod_label, yaml: pod) { stage('Kola:QEMU basic') { utils.shwrap(""" cosa kola run --basic-qemu-scenarios --no-test-exit-error - tar -cf - tmp/kola/ | xz -c9 > kola-basic.tar.xz + tar -cf - tmp/kola/ | xz -c9 > kola-run-basic.tar.xz """) - archiveArtifacts "kola-basic.tar.xz" + archiveArtifacts "kola-run-basic.tar.xz" } - if (!checkKolaSuccess(utils, currentBuild)) { + if (!utils.checkKolaSuccess("tmp/kola", currentBuild)) { return } @@ -313,11 +299,22 @@ podTemplate(cloud: 'openshift', label: pod_label, yaml: pod) { def parallel = ((cosa_memory_request_mb - 512) / 1024) as Integer utils.shwrap(""" coreos-assembler kola run --parallel ${parallel} --no-test-exit-error - tar -cf - tmp/kola/ | xz -c9 > _kola_temp.tar.xz + tar -cf - tmp/kola/ | xz -c9 > kola-run.tar.xz + """) + archiveArtifacts "kola-run.tar.xz" + } + if (!utils.checkKolaSuccess("tmp/kola", currentBuild)) { + return + } + + stage('Kola:QEMU upgrade') { + utils.shwrap(""" + coreos-assembler kola --upgrades --no-test-exit-error + tar -cf - tmp/kola/ | xz -c9 > kola-run-upgrade.tar.xz """) - archiveArtifacts "_kola_temp.tar.xz" + archiveArtifacts "kola-run-upgrade.tar.xz" } - if (!checkKolaSuccess(utils, currentBuild)) { + if (!utils.checkKolaSuccess("tmp/kola", currentBuild)) { return } diff --git a/Jenkinsfile.kola.aws b/Jenkinsfile.kola.aws index 666187b5b..b23e94738 100644 --- a/Jenkinsfile.kola.aws +++ b/Jenkinsfile.kola.aws @@ -40,6 +40,7 @@ podTemplate(cloud: 'openshift', label: pod_label, yaml: pod) { def ami, ami_region def no_ami = false + def meta_json stage('Fetch Metadata') { utils.shwrap(""" export AWS_CONFIG_FILE=\${AWS_FCOS_BUILDS_BOT_CONFIG} @@ -48,7 +49,7 @@ podTemplate(cloud: 'openshift', label: pod_label, yaml: pod) { """) def basearch = utils.shwrap_capture("coreos-assembler basearch") - def meta_json = "builds/${params.VERSION}/${basearch}/meta.json" + meta_json = "builds/${params.VERSION}/${basearch}/meta.json" def meta = readJSON file: meta_json if (meta.amis.size() > 0) { ami = meta['amis'][0]['hvm'] @@ -64,20 +65,33 @@ podTemplate(cloud: 'openshift', label: pod_label, yaml: pod) { return } - stage('AWS Kola Run') { - utils.shwrap(""" - export AWS_CONFIG_FILE=\${AWS_FCOS_KOLA_BOT_CONFIG} - # use `cosa kola` here since it knows about blacklisted tests - cosa kola -- run -p aws --aws-ami ${ami} --aws-region ${ami_region} -b fcos -j 5 || : - tar -cf - tmp/kola | xz -c9 > kola_tmp.tar.xz - """) - archiveArtifacts "kola_tmp.tar.xz" - } - - def report = readJSON file: "tmp/kola/reports/report.json" - if (report["result"] != "PASS") { - currentBuild.result = 'FAILURE' - return + stage('Kola') { + parallel aws: { + stage('Kola:AWS') { + utils.shwrap(""" + export AWS_CONFIG_FILE=\${AWS_FCOS_KOLA_BOT_CONFIG} + # use `cosa kola` here since it knows about blacklisted tests + cosa kola run -p aws --aws-ami ${ami} --aws-region ${ami_region} -b fcos -j 5 --no-test-exit-error + tar -cf - tmp/kola | xz -c9 > kola-run.tar.xz + """) + archiveArtifacts "kola-run.tar.xz" + } + if (!utils.checkKolaSuccess("tmp/kola", currentBuild)) { + return + } + }, + aws_upgrade: { + stage('Kola:AWS upgrade') { + utils.shwrap(""" + coreos-assembler kola --upgrades -p=aws --aws-region ${ami_region} --no-test-exit-error + tar -cf - tmp/kola/ | xz -c9 > kola-run-upgrade.tar.xz + """) + archiveArtifacts "kola-run-upgrade.tar.xz" + } + if (!utils.checkKolaSuccess("tmp/kola-upgrade", currentBuild)) { + return + } + } } }} } diff --git a/utils.groovy b/utils.groovy index 4fdabd687..17c8f5cc5 100644 --- a/utils.groovy +++ b/utils.groovy @@ -43,6 +43,21 @@ def path_exists(path) { return shwrap_rc("test -e ${path}") == 0 } +// Parse and handle the result of Kola +boolean checkKolaSuccess(dir, currentBuild) { + // archive the image if the tests failed + def report = readJSON file: "${dir}/reports/report.json" + def result = report["result"] + print("kola result: ${result}") + if (result != "PASS" && report["platform"] == "qemu-unpriv") { + shwrap("coreos-assembler compress --compressor xz") + archiveArtifacts "builds/latest/**/*.qcow2.xz" + currentBuild.result = 'FAILURE' + return false + } + return true +} + def aws_s3_cp_allow_noent(src, dest) { // see similar code in `cosa buildprep` shwrap("""