Skip to content

Commit

Permalink
[FTSR] Convert to tasks and add jest/api integration suites (#91770)
Browse files Browse the repository at this point in the history
  • Loading branch information
@brianseeders @kibanamachine
brianseeders authored and kibanamachine committed Feb 18, 2021
1 parent cab949b commit c7af426
Show file tree
Hide file tree
Showing 2 changed files with 85 additions and 63 deletions.
137 changes: 78 additions & 59 deletions .ci/Jenkinsfile_flaky
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,47 +3,39 @@
library 'kibana-pipeline-library'
kibanaLibrary.load()

def CI_GROUP_PARAM = params.CI_GROUP
def TASK_PARAM = params.TASK ?: params.CI_GROUP

// Looks like 'oss:ciGroup:1', 'oss:firefoxSmoke'
def JOB_PARTS = CI_GROUP_PARAM.split(':')
def JOB_PARTS = TASK_PARAM.split(':')
def IS_XPACK = JOB_PARTS[0] == 'xpack'
def JOB = JOB_PARTS[1]
def JOB = JOB_PARTS.size() > 1 ? JOB_PARTS[1] : JOB_PARTS[0]
def CI_GROUP = JOB_PARTS.size() > 2 ? JOB_PARTS[2] : ''
def EXECUTIONS = params.NUMBER_EXECUTIONS.toInteger()
def AGENT_COUNT = getAgentCount(EXECUTIONS)

def worker = getWorkerFromParams(IS_XPACK, JOB, CI_GROUP)

def workerFailures = []
def NEED_BUILD = JOB != 'jestIntegration' && JOB != 'apiIntegration'

currentBuild.displayName += trunc(" ${params.GITHUB_OWNER}:${params.branch_specifier}", 24)
currentBuild.description = "${params.CI_GROUP}<br />Agents: ${AGENT_COUNT}<br />Executions: ${params.NUMBER_EXECUTIONS}"

kibanaPipeline(timeoutMinutes: 180) {
def agents = [:]
def workerFailures = []

def worker = getWorkerFromParams(IS_XPACK, JOB, CI_GROUP)

for(def agentNumber = 1; agentNumber <= AGENT_COUNT; agentNumber++) {
def agentNumberInside = agentNumber
def agentExecutions = floor(EXECUTIONS/AGENT_COUNT) + (agentNumber <= EXECUTIONS%AGENT_COUNT ? 1 : 0)

agents["agent-${agentNumber}"] = {
catchErrors {
print "Agent ${agentNumberInside} - ${agentExecutions} executions"

withEnv([
'IGNORE_SHIP_CI_STATS_ERROR=true',
]) {
workers.functional('flaky-test-runner', {
if (!IS_XPACK) {
kibanaPipeline.buildOss()
if (CI_GROUP == '1') {
runbld("./test/scripts/jenkins_build_kbn_sample_panel_action.sh", "Build kbn tp sample panel action for ciGroup1")
}
} else {
kibanaPipeline.buildXpack()
}
}, getWorkerMap(agentNumberInside, agentExecutions, worker, workerFailures))()
}
}
agentProcess(
agentNumber: agentNumber,
agentExecutions: agentExecutions,
worker: worker,
workerFailures: workerFailures,
needBuild: NEED_BUILD,
isXpack: IS_XPACK,
ciGroup: CI_GROUP
)
}
}

Expand All @@ -59,14 +51,70 @@ kibanaPipeline(timeoutMinutes: 180) {
}
}

def agentProcess(Map params = [:]) {
def config = [
agentNumber: 1,
agentExecutions: 0,
worker: {},
workerFailures: [],
needBuild: false,
isXpack: false,
ciGroup: null,
] + params

catchErrors {
print "Agent ${config.agentNumber} - ${config.agentExecutions} executions"

withEnv([
'IGNORE_SHIP_CI_STATS_ERROR=true',
]) {
kibanaPipeline.withTasks([
parallel: 20,
]) {
task {
if (config.needBuild) {
if (!config.isXpack) {
kibanaPipeline.buildOss()
} else {
kibanaPipeline.buildXpack()
}
}

for(def i = 0; i < config.agentExecutions; i++) {
def taskNumber = i
task({
withEnv([
"REMOVE_KIBANA_INSTALL_DIR=1",
]) {
catchErrors {
try {
config.worker()
} catch (ex) {
config.workerFailures << "agent-${config.agentNumber}-${taskNumber}"
throw ex
}
}
}
})
}
}
}
}
}
}

def getWorkerFromParams(isXpack, job, ciGroup) {
if (!isXpack) {
if (job == 'accessibility') {
return kibanaPipeline.functionalTestProcess('kibana-accessibility', './test/scripts/jenkins_accessibility.sh')
} else if (job == 'firefoxSmoke') {
return kibanaPipeline.functionalTestProcess('firefoxSmoke', './test/scripts/jenkins_firefox_smoke.sh')
} else if(job == 'visualRegression') {
} else if (job == 'visualRegression') {
return kibanaPipeline.functionalTestProcess('visualRegression', './test/scripts/jenkins_visual_regression.sh')
} else if (job == 'jestIntegration') {
return kibanaPipeline.scriptTaskDocker('Jest Integration Tests', 'test/scripts/test/jest_integration.sh')
} else if (job == 'apiIntegration') {
return kibanaPipeline.scriptTask('API Integration Tests', 'test/scripts/test/api_integration.sh')
} else {
return kibanaPipeline.ossCiGroupProcess(ciGroup)
}
Expand All @@ -76,45 +124,16 @@ def getWorkerFromParams(isXpack, job, ciGroup) {
return kibanaPipeline.functionalTestProcess('xpack-accessibility', './test/scripts/jenkins_xpack_accessibility.sh')
} else if (job == 'firefoxSmoke') {
return kibanaPipeline.functionalTestProcess('xpack-firefoxSmoke', './test/scripts/jenkins_xpack_firefox_smoke.sh')
} else if(job == 'visualRegression') {
} else if (job == 'visualRegression') {
return kibanaPipeline.functionalTestProcess('xpack-visualRegression', './test/scripts/jenkins_xpack_visual_regression.sh')
} else {
return kibanaPipeline.xpackCiGroupProcess(ciGroup)
}
}

def getWorkerMap(agentNumber, numberOfExecutions, worker, workerFailures, maxWorkerProcesses = 12) {
def workerMap = [:]
def numberOfWorkers = Math.min(numberOfExecutions, maxWorkerProcesses)

for(def i = 1; i <= numberOfWorkers; i++) {
def workerExecutions = floor(numberOfExecutions/numberOfWorkers + (i <= numberOfExecutions%numberOfWorkers ? 1 : 0))

workerMap["agent-${agentNumber}-worker-${i}"] = { workerNumber ->
for(def j = 0; j < workerExecutions; j++) {
print "Execute agent-${agentNumber} worker-${workerNumber}: ${j}"
withEnv([
"REMOVE_KIBANA_INSTALL_DIR=1",
]) {
catchErrors {
try {
worker(workerNumber)
} catch (ex) {
workerFailures << "agent-${agentNumber} worker-${workerNumber}-${j}"
throw ex
}
}
}
}
}
}

return workerMap
}

def getAgentCount(executions) {
// Increase agent count every 24 worker processess, up to 3 agents maximum
return Math.min(3, 1 + floor(executions/24))
// Increase agent count every 20 worker processess, up to 3 agents maximum
return Math.min(3, 1 + floor(executions/20))
}

def trunc(str, length) {
Expand Down
11 changes: 7 additions & 4 deletions vars/kibanaPipeline.groovy
View file
Original file line number Diff line number Diff line change
Expand Up @@ -408,12 +408,13 @@ def buildXpackPlugins() {
runbld('./test/scripts/jenkins_xpack_build_plugins.sh', 'Build X-Pack Plugins')
}

def withTasks(Map params = [worker: [:]], Closure closure) {
def withTasks(Map params = [:], Closure closure) {
catchErrors {
def config = [name: 'ci-worker', size: 'xxl', ramDisk: true] + (params.worker ?: [:])
def config = [setupWork: {}, worker: [:], parallel: 24] + params
def workerConfig = [name: 'ci-worker', size: 'xxl', ramDisk: true] + config.worker

workers.ci(config) {
withCiTaskQueue(parallel: 24) {
workers.ci(workerConfig) {
withCiTaskQueue([parallel: config.parallel]) {
parallel([
docker: {
retry(2) {
Expand All @@ -426,6 +427,8 @@ def withTasks(Map params = [worker: [:]], Closure closure) {
xpackPlugins: { buildXpackPlugins() },
])

config.setupWork()

catchErrors {
closure()
}
Expand Down

0 comments on commit c7af426

Please sign in to comment.