Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

[CI] Optimise stash/unstash performance #18473

Merged
merged 14 commits into from
May 14, 2020
Merged

[CI] Optimise stash/unstash performance #18473

Changes from 2 commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
fbd8fb8
[CI] Optimise stash/unstash performance
v1v May 13, 2020
86ac4c5
[CI] Use google storage
v1v May 13, 2020
0ec32a7
ci: flatten files
v1v May 13, 2020
1b612aa
ci: fix the permission denied after the unzip
v1v May 13, 2020
04c2db7
ci: let's be explicit
v1v May 13, 2020
44a5679
ci: zip does not compress the .git folder
v1v May 13, 2020
d4b1421
ci: fix the permission denied after the unzip
v1v May 13, 2020
3563a8d
ci: use tar to support restore permissions
v1v May 13, 2020
df947c2
ci: fix the file changed as we read it
v1v May 13, 2020
8dfa353
Revert "ci: fix the file changed as we read it"
v1v May 13, 2020
ba137ae
ci: fix the file changed as we read it
v1v May 13, 2020
4a8ceec
Tweak some of the stash/unstash arguments
v1v May 14, 2020
aabc3de
ci: refactored with the shared library steps
v1v May 14, 2020
0104e63
Update Jenkinsfile
v1v May 14, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
36 changes: 29 additions & 7 deletions Jenkinsfile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,8 @@ pipeline {
DOCKER_REGISTRY = 'docker.elastic.co'
AWS_ACCOUNT_SECRET = 'secret/observability-team/ci/elastic-observability-aws-account-auth'
RUNBLD_DISABLE_NOTIFICATIONS = 'true'
JOB_GCS_BUCKET = 'beats-ci-temp'
v1v marked this conversation as resolved.
Show resolved Hide resolved
JOB_GCS_CREDENTIALS = 'beats-ci-gcs-plugin'
}
options {
timeout(time: 2, unit: 'HOURS')
Expand Down Expand Up @@ -57,7 +59,7 @@ pipeline {
pipelineManager([ cancelPreviousRunningBuilds: [ when: 'PR' ] ])
deleteDir()
gitCheckout(basedir: "${BASE_DIR}", githubNotifyFirstTimeContributor: true)
stash allowEmpty: true, name: 'source', useDefaultExcludes: false
stashV2(allowEmpty: true, name: 'source', useDefaultExcludes: false)
v1v marked this conversation as resolved.
Show resolved Hide resolved
dir("${BASE_DIR}"){
loadConfigEnvVars()
}
Expand Down Expand Up @@ -722,7 +724,7 @@ def withBeatsEnv(boolean archive, Closure body) {
"DOCKER_PULL=0",
]) {
deleteDir()
unstash 'source'
unstashV2('source')
if(isDockerInstalled()){
dockerLogin(secret: "${DOCKERELASTIC_SECRET}", registry: "${DOCKER_REGISTRY}")
}
Expand Down Expand Up @@ -763,7 +765,7 @@ def withBeatsEnvWin(Closure body) {
"RACE_DETECTOR=true",
]){
deleteDir()
unstash 'source'
unstashV2('source')
dir("${env.BASE_DIR}"){
installTools()
try {
Expand Down Expand Up @@ -1015,7 +1017,7 @@ def startCloudTestEnv(String name, environments = []) {
// Archive terraform states in case manual cleanup is needed.
archiveArtifacts(allowEmptyArchive: true, artifacts: '**/terraform.tfstate')
}
stash(name: "terraform-${name}", allowEmpty: true, includes: '**/terraform.tfstate,**/.terraform/**')
stashV2(name: "terraform-${name}", allowEmpty: true, includes: '**/terraform.tfstate,**/.terraform/**')
}
}
}
Expand All @@ -1027,7 +1029,7 @@ def terraformCleanup(String stashName, String directory) {
stage("Remove cloud scenarios in ${directory}"){
withCloudTestEnv() {
withBeatsEnv(false) {
unstash "terraform-${stashName}"
unstashV2("terraform-${stashName}")
retry(2) {
sh(label: "Terraform Cleanup", script: ".ci/scripts/terraform-cleanup.sh ${directory}")
}
Expand Down Expand Up @@ -1163,7 +1165,7 @@ def junitAndStore(Map params = [:]){
junit(params)
// STAGE_NAME env variable could be null in some cases, so let's use the currentmilliseconds
def stageName = env.STAGE_NAME ? env.STAGE_NAME.replaceAll("[\\W]|_",'-') : "uncategorized-${new java.util.Date().getTime()}"
stash(includes: params.testResults, allowEmpty: true, name: stageName, useDefaultExcludes: true)
stashV2(includes: params.testResults, allowEmpty: true, name: stageName, useDefaultExcludes: true)
v1v marked this conversation as resolved.
Show resolved Hide resolved
stashedTestReports[stageName] = stageName
}

Expand All @@ -1176,7 +1178,7 @@ def runbld() {
// Unstash the test reports
stashedTestReports.each { k, v ->
dir(k) {
unstash v
unstashV2(v)
}
}
sh(label: 'Process JUnit reports with runbld',
Expand All @@ -1190,3 +1192,23 @@ def runbld() {
}
}
}

def stashV2(Map args = [:]) {
zip dir: '', glob: '', zipFile: "${args.name}.tgz"
googleStorageUpload(
bucket: "gs://${JOB_GCS_BUCKET}/${JOB_NAME}-${BUILD_NUMBER}/${args.name}",
credentialsId: "${JOB_GCS_CREDENTIALS}",
pattern: "${args.name}.tgz",
sharedPublicly: false,
showInline: true
)
v1v marked this conversation as resolved.
Show resolved Hide resolved
}

def unstashV2(String id) {
googleStorageDownload(
bucketUri: "gs://${JOB_GCS_BUCKET}/${JOB_NAME}-${BUILD_NUMBER}/${id}/${id}.tgz",
credentialsId: "${JOB_GCS_CREDENTIALS}",
localDirectory: ''
)
unzip dir: '', glob: '', quiet: true, zipFile: "${id}.tgz"
v1v marked this conversation as resolved.
Show resolved Hide resolved
}