From 1f74b94e6840a02572799c4d8732a931144970f3 Mon Sep 17 00:00:00 2001 From: Victor Martinez Date: Thu, 16 Jul 2020 11:24:13 +0100 Subject: [PATCH] [CI] fix MODULE variable cornercases --- Jenkinsfile | 64 ++++++++++++++++++++++++++++++----------------------- 1 file changed, 36 insertions(+), 28 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index e0f12058de46..b32b1b74a614 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -153,7 +153,7 @@ pipeline { } } steps { - makeTarget(context: "Filebeat oss Linux", target: "-C filebeat testsuite", withModule: true) + makeTarget(context: "Filebeat oss Linux", directory: 'filebeat', target: 'testsuite', withModule: true) } } stage('Filebeat x-pack'){ @@ -243,7 +243,7 @@ pipeline { stages { stage('Heartbeat oss'){ steps { - makeTarget(context: "Heartbeat oss Linux", target: "-C heartbeat testsuite") + makeTarget(context: "Heartbeat oss Linux", directory: 'heartbeat', target: "testsuite") } } stage('Heartbeat Mac OS X'){ @@ -289,7 +289,7 @@ pipeline { } } steps { - makeTarget(context: "Auditbeat oss Linux", target: "-C auditbeat testsuite", withModule: true) + makeTarget(context: "Auditbeat oss Linux", directory: 'auditbeat', target: "testsuite", withModule: true) } } stage('Auditbeat crosscompile'){ @@ -302,7 +302,7 @@ pipeline { } } steps { - makeTarget(context: "Auditbeat oss crosscompile", target: "-C auditbeat crosscompile") + makeTarget(context: "Auditbeat oss crosscompile", directory: 'auditbeat', target: "crosscompile") } } stage('Auditbeat oss Mac OS X'){ @@ -387,12 +387,12 @@ pipeline { stages { stage('Libbeat oss'){ steps { - makeTarget(context: "Libbeat oss Linux", target: "-C libbeat testsuite") + makeTarget(context: "Libbeat oss Linux", directory: 'libbeat', target: "testsuite") } } stage('Libbeat crosscompile'){ steps { - makeTarget(context: "Libbeat oss crosscompile", target: "-C libbeat crosscompile") + makeTarget(context: "Libbeat oss crosscompile", directory: 'libbeat', target: "crosscompile") } } stage('Libbeat stress-tests'){ @@ -412,7 +412,7 @@ pipeline { } } steps { - makeTarget(context: "Libbeat x-pack Linux", target: "-C x-pack/libbeat testsuite") + makeTarget(context: "Libbeat x-pack Linux", directory: 'x-pack/libbeat', target: "testsuite") } } stage('Metricbeat OSS Unit tests'){ @@ -499,7 +499,7 @@ pipeline { } } steps { - makeTarget(context: "Metricbeat OSS crosscompile", target: "-C metricbeat crosscompile") + makeTarget(context: "Metricbeat OSS crosscompile", directory: 'metricbeat', target: "crosscompile") } } stage('Metricbeat Mac OS X'){ @@ -571,7 +571,7 @@ pipeline { stages { stage('Packetbeat oss'){ steps { - makeTarget(context: "Packetbeat oss Linux", target: "-C packetbeat testsuite") + makeTarget(context: "Packetbeat oss Linux", directory: 'packetbeat', target: "testsuite") } } } @@ -605,7 +605,7 @@ pipeline { stages { stage('Winlogbeat oss'){ steps { - makeTarget(context: "Winlogbeat oss crosscompile", target: "-C winlogbeat crosscompile") + makeTarget(context: "Winlogbeat oss crosscompile", directory: 'winlogbeat', target: "crosscompile") } } stage('Winlogbeat Windows'){ @@ -650,7 +650,7 @@ pipeline { steps { mageTarget(context: "Functionbeat x-pack Linux", directory: "x-pack/functionbeat", target: "update build test") withEnv(["GO_VERSION=1.13.1"]){ - makeTarget(context: "Functionbeat x-pack Linux", target: "-C x-pack/functionbeat test-gcp-functions") + makeTarget(context: "Functionbeat x-pack Linux", directory: 'x-pack/functionbeat', target: "test-gcp-functions") } } } @@ -699,7 +699,7 @@ pipeline { stages { stage('Journalbeat oss'){ steps { - makeTarget(context: "Journalbeat Linux", target: "-C journalbeat testsuite") + makeTarget(context: "Journalbeat Linux", directory: 'journalbeat', target: "testsuite") } } } @@ -716,14 +716,14 @@ pipeline { stages { stage('Generators Metricbeat Linux'){ steps { - makeTarget(context: "Generators Metricbeat Linux", target: "-C generator/_templates/metricbeat test") - makeTarget(context: "Generators Metricbeat Linux", target: "-C generator/_templates/metricbeat test-package") + makeTarget(context: "Generators Metricbeat Linux", directory: 'generator/_templates/metricbeat', target: "test") + makeTarget(context: "Generators Metricbeat Linux", directory: 'generator/_templates/metricbeat', target: "test-package") } } stage('Generators Beat Linux'){ steps { - makeTarget(context: "Generators Beat Linux", target: "-C generator/_templates/beat test") - makeTarget(context: "Generators Beat Linux", target: "-C generator/_templates/beat test-package") + makeTarget(context: "Generators Beat Linux", directory: 'generator/_templates/beat', target: "test") + makeTarget(context: "Generators Beat Linux", directory: 'generator/_templates/beat', target: "test-package") } } stage('Generators Metricbeat Mac OS X'){ @@ -736,7 +736,7 @@ pipeline { } } steps { - makeTarget(context: "Generators Metricbeat Mac OS X", target: "-C generator/_templates/metricbeat test") + makeTarget(context: "Generators Metricbeat Mac OS X", directory: 'generator/_templates/metricbeat', target: "test") } post { always { @@ -754,7 +754,7 @@ pipeline { } } steps { - makeTarget(context: "Generators Beat Mac OS X", target: "-C generator/_templates/beat test") + makeTarget(context: "Generators Beat Mac OS X", directory: 'generator/_templates/beat', target: "test") } post { always { @@ -807,15 +807,17 @@ def fixPermissions(location) { def makeTarget(Map args = [:]) { def context = args.context def target = args.target + def directory = args.get('directory', '') def clean = args.get('clean', true) def withModule = args.get('withModule', false) + def directoryFlag = directory.trim() ? "-C ${directory}" : '' withGithubNotify(context: "${context}") { - withBeatsEnv(archive: true, withModule: withModule, modulePattern: getModulePattern(target)) { + withBeatsEnv(archive: true, withModule: withModule, directory: directory) { whenTrue(params.debug) { dumpFilteredEnvironment() dumpMage() } - sh(label: "Make ${target}", script: "make ${target}") + sh(label: "Make ${target}", script: "make ${directoryFlag} ${target}") whenTrue(clean) { fixPermissions("${HOME}") } @@ -829,7 +831,7 @@ def mageTarget(Map args = [:]) { def target = args.target def withModule = args.get('withModule', false) withGithubNotify(context: "${context}") { - withBeatsEnv(archive: true, withModule: withModule, modulePattern: getModulePattern(directory)) { + withBeatsEnv(archive: true, withModule: withModule, directory: directory) { whenTrue(params.debug) { dumpFilteredEnvironment() dumpMage() @@ -849,7 +851,7 @@ def mageTargetWin(Map args = [:]) { def target = args.target def withModule = args.get('withModule', false) withGithubNotify(context: "${context}") { - withBeatsEnvWin(withModule: withModule, modulePattern: getModulePattern(directory)) { + withBeatsEnvWin(withModule: withModule, directory: directory) { whenTrue(params.debug) { dumpFilteredEnvironment() dumpMageWin() @@ -871,9 +873,10 @@ def getModulePattern(String toCompare) { def withBeatsEnv(Map args = [:], Closure body) { def archive = args.get('archive', true) def withModule = args.get('withModule', false) + def directory = args.get('directory', '') def modulePattern if (withModule) { - modulePattern = args.containsKey('modulePattern') ? args.modulePattern : error('withBeatsEnv: modulePattern parameter is required.') + modulePattern = getModulePattern(directory) } def os = goos() def goRoot = "${env.WORKSPACE}/.gvm/versions/go${GO_VERSION}.${os}.amd64" @@ -882,7 +885,7 @@ def withBeatsEnv(Map args = [:], Closure body) { unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") // NOTE: This is required to run after the unstash - def module = withModule ? getCommonModuleInTheChangeSet(modulePattern) : '' + def module = withModule ? getCommonModuleInTheChangeSet(modulePattern, directory) : '' withEnv([ "HOME=${env.WORKSPACE}", @@ -924,9 +927,10 @@ def withBeatsEnv(Map args = [:], Closure body) { def withBeatsEnvWin(Map args = [:], Closure body) { def withModule = args.get('withModule', false) + def directory = args.get('directory', '') def modulePattern if (withModule) { - modulePattern = args.containsKey('modulePattern') ? args.modulePattern : error('withBeatsEnvWin: modulePattern parameter is required.') + modulePattern = getModulePattern(directory) } final String chocoPath = 'C:\\ProgramData\\chocolatey\\bin' final String chocoPython3Path = 'C:\\Python38;C:\\Python38\\Scripts' @@ -936,7 +940,7 @@ def withBeatsEnvWin(Map args = [:], Closure body) { unstashV2(name: 'source', bucket: "${JOB_GCS_BUCKET}", credentialsId: "${JOB_GCS_CREDENTIALS}") // NOTE: This is required to run after the unstash - def module = withModule ? getCommonModuleInTheChangeSet(modulePattern) : '' + def module = withModule ? getCommonModuleInTheChangeSet(modulePattern, directory) : '' withEnv([ "HOME=${env.WORKSPACE}", @@ -1302,10 +1306,14 @@ def loadConfigEnvVars(){ For such, it's required to look for changes under the module folder and exclude anything else such as ascidoc and png files. */ -def getCommonModuleInTheChangeSet(String pattern) { +def getCommonModuleInTheChangeSet(String pattern, String directory) { def module = '' + // Transform folder structure in regex format since path separator is required to be escaped + def transformedDirectory = directory.replaceAll('/', '\\/') + def directoryExclussion = "((?!^${transformedDirectory}\\/).)*\$" + def exclude = "^(${directoryExclussion}|((?!\\/module\\/).)*\$|.*\\.asciidoc|.*\\.png)" dir("${env.BASE_DIR}") { - module = getGitMatchingGroup(pattern: pattern , exclude: '^(((?!\\/module\\/).)*$|.*\\.asciidoc|.*\\.png)') + module = getGitMatchingGroup(pattern: pattern, exclude: exclude) } return module }