From 49729c2a72265b9cb44485c9a37d0630282a7c87 Mon Sep 17 00:00:00 2001 From: Manvenddra Rawat Date: Wed, 12 Apr 2023 10:31:36 +0530 Subject: [PATCH 1/6] Fixed dockerfile change detection to rebuild docker image Signed-off-by: Manvenddra Rawat --- Jenkins/Jenkinsfile | 33 +++++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/Jenkins/Jenkinsfile b/Jenkins/Jenkinsfile index 18b07cc4ba7..d081b02eb7b 100755 --- a/Jenkins/Jenkinsfile +++ b/Jenkins/Jenkinsfile @@ -72,10 +72,14 @@ pipeline { } sh "bash -l -c \"git clone ${env.REPO_OR_FORK_URL} -b ${env.CHANGE_BRANCH} commit_checks_repo\"" if ("${env.QCInternalValidation}" == "true") { + if (didDockerFileChange("tf-gpu")) { + env.USE_LINARO="" + } withCredentials([usernameColonPassword(credentialsId: 'neoci-pat', variable: 'USERPASS')]) { sh "AIMET_VARIANT=\"tf-gpu\" bash -l -c \"cd ${REPO_NAME} && ${env.DEPENDENCY_DATA_PATH}/commit_check/commit-checks.sh -e AIMET_VARIANT ${env.USE_LINARO} ${env.PREBUILT_DOCKER_IMAGE_URL}\"" } } else { + env.USE_LINARO="" sh "AIMET_VARIANT=\"tf-gpu\" bash -l -c \"cd ${REPO_NAME} && ${env.DEPENDENCY_DATA_PATH}/commit_check/commit-checks.sh -e AIMET_VARIANT ${env.USE_LINARO} ${env.PREBUILT_DOCKER_IMAGE_URL}\"" } } @@ -611,6 +615,19 @@ pipeline { } } +def didDockerFileChange(aimet_variant) { + def changedFiles = pullRequest.files.collect { + it.getFilename() + } + print changedFiles + if (changedFiles.contains("Jenkins/Dockerfile.${aimet_variant}".toString())) { + print "Jenkins/Dockerfile.${aimet_variant} changed in PR, so building docker image locally." + return true + } + print "Jenkins/Dockerfile.${aimet_variant} NOT changed in PR, so using pre-built docker image." + return false +} + def runStage(aimet_variant, options) { echo "*** Running stage ${options} for ${aimet_variant} variant on ${env.NODE_NAME} in workspace ${env.WORKSPACE_ROOT} ***" @@ -622,13 +639,10 @@ def runStage(aimet_variant, options) { } else { - def changedFiles = pullRequest.files.collect { - it.getFilename() - } - print changedFiles - if (!changedFiles.contains("Jenkins/Dockerfile.${aimet_variant}".toString())) { - print "Jenkins/Dockerfile.${aimet_variant} not found in changed file list, so using Linaro Docker image for ${aimet_variant}" + if (didDockerFileChange(env.aimet_variant)) { + env.USE_LINARO="" } + echo "*** with linaro option as ${env.USE_LINARO} ***" sh """ AIMET_VARIANT=${aimet_variant} bash -l -c "cd ${REPO_NAME} && ./buildntest.sh -e AIMET_VARIANT ${options} ${env.USE_LINARO} ${env.PREBUILT_DOCKER_IMAGE_URL}" """ @@ -637,12 +651,15 @@ def runStage(aimet_variant, options) { def callAimetExtra(target_branch) { echo target_branch + if (didDockerFileChange("tf-gpu") || didDockerFileChange("tf-cpu") || didDockerFileChange("torch-gpu") || didDockerFileChange("torch-cpu")) { + env.USE_LINARO="" + } if (target_branch != "develop") { echo "Running AIMET additional stages on ${CHANGE_TARGET} branch ..." - build job: "AIMET-Extra", parameters: [string(name: 'AIMET_GIT_COMMIT', value: "${CHANGE_BRANCH}"), string(name: 'PROJECT_BRANCH', value: target_branch)] + build job: "AIMET-Extra", parameters: [string(name: 'AIMET_GIT_COMMIT', value: "${CHANGE_BRANCH}"), string(name: 'PROJECT_BRANCH', value: target_branch), string(name: 'USE_LINARO', value: "${env.USE_LINARO}")] } else { echo "Running AIMET additional stages on develop branch ..." - build job: "AIMET-Extra", parameters: [string(name: 'AIMET_GIT_COMMIT', value: "${CHANGE_BRANCH}")] + build job: "AIMET-Extra", parameters: [string(name: 'AIMET_GIT_COMMIT', value: "${CHANGE_BRANCH}"), string(name: 'USE_LINARO', value: "${env.USE_LINARO}")] } } From 59a7b5b333c931549ee8bb676f941644d5cb5c3a Mon Sep 17 00:00:00 2001 From: Manvenddra Rawat Date: Thu, 13 Apr 2023 22:00:42 +0530 Subject: [PATCH 2/6] Fixed Jenkinsfile to use BUILD_TIMEOUT & added comments Signed-off-by: Manvenddra Rawat --- Jenkins/Jenkinsfile | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/Jenkins/Jenkinsfile b/Jenkins/Jenkinsfile index d081b02eb7b..36df39d8b76 100755 --- a/Jenkins/Jenkinsfile +++ b/Jenkins/Jenkinsfile @@ -10,7 +10,7 @@ pipeline { agent { label "${params.BUILD_LABEL_CPU}" } options { timestamps() - timeout( time: 3, unit: 'HOURS' ) + timeout( time: "${BUILD_TIMEOUT}", unit: 'HOURS' ) } environment{ def workspace = pwd() @@ -73,12 +73,14 @@ pipeline { sh "bash -l -c \"git clone ${env.REPO_OR_FORK_URL} -b ${env.CHANGE_BRANCH} commit_checks_repo\"" if ("${env.QCInternalValidation}" == "true") { if (didDockerFileChange("tf-gpu")) { + // setting USE LINARO value to EMPTY to rebuild docker image env.USE_LINARO="" } withCredentials([usernameColonPassword(credentialsId: 'neoci-pat', variable: 'USERPASS')]) { sh "AIMET_VARIANT=\"tf-gpu\" bash -l -c \"cd ${REPO_NAME} && ${env.DEPENDENCY_DATA_PATH}/commit_check/commit-checks.sh -e AIMET_VARIANT ${env.USE_LINARO} ${env.PREBUILT_DOCKER_IMAGE_URL}\"" } } else { + // setting USE LINARO value to EMPTY to rebuild docker image env.USE_LINARO="" sh "AIMET_VARIANT=\"tf-gpu\" bash -l -c \"cd ${REPO_NAME} && ${env.DEPENDENCY_DATA_PATH}/commit_check/commit-checks.sh -e AIMET_VARIANT ${env.USE_LINARO} ${env.PREBUILT_DOCKER_IMAGE_URL}\"" } @@ -619,12 +621,12 @@ def didDockerFileChange(aimet_variant) { def changedFiles = pullRequest.files.collect { it.getFilename() } - print changedFiles + echo "Changed files - ${changedFiles}" if (changedFiles.contains("Jenkins/Dockerfile.${aimet_variant}".toString())) { - print "Jenkins/Dockerfile.${aimet_variant} changed in PR, so building docker image locally." + echo "*** Jenkins/Dockerfile.${aimet_variant} changed in PR, so building docker image locally. ***" return true } - print "Jenkins/Dockerfile.${aimet_variant} NOT changed in PR, so using pre-built docker image." + echo "*** Jenkins/Dockerfile.${aimet_variant} NOT changed in PR, so using pre-built docker image. ***" return false } @@ -640,9 +642,9 @@ def runStage(aimet_variant, options) { } else { if (didDockerFileChange(env.aimet_variant)) { + // setting USE LINARO value to EMPTY to rebuild docker image env.USE_LINARO="" } - echo "*** with linaro option as ${env.USE_LINARO} ***" sh """ AIMET_VARIANT=${aimet_variant} bash -l -c "cd ${REPO_NAME} && ./buildntest.sh -e AIMET_VARIANT ${options} ${env.USE_LINARO} ${env.PREBUILT_DOCKER_IMAGE_URL}" """ @@ -652,6 +654,7 @@ def runStage(aimet_variant, options) { def callAimetExtra(target_branch) { echo target_branch if (didDockerFileChange("tf-gpu") || didDockerFileChange("tf-cpu") || didDockerFileChange("torch-gpu") || didDockerFileChange("torch-cpu")) { + // setting USE LINARO value to EMPTY to rebuild docker image env.USE_LINARO="" } if (target_branch != "develop") { From 4c2a8ed6f8d6a1aec4ff78195906dbdc35872098 Mon Sep 17 00:00:00 2001 From: Manvenddra Rawat Date: Sat, 15 Apr 2023 17:04:05 +0530 Subject: [PATCH 3/6] Fix rename of base image in Dockerfile.torch-cpu Signed-off-by: Manvenddra Rawat --- Jenkins/Dockerfile.torch-cpu | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkins/Dockerfile.torch-cpu b/Jenkins/Dockerfile.torch-cpu index b388a01cbf5..394dc1a0ded 100644 --- a/Jenkins/Dockerfile.torch-cpu +++ b/Jenkins/Dockerfile.torch-cpu @@ -37,7 +37,7 @@ # Docker image file to build and test AIMET for PyTorch in a CPU environment ARG REGISTRY -FROM ${REGISTRY}ubuntu:bionic as target +FROM ${REGISTRY}ubuntu:bionic ARG DEBIAN_FRONTEND=noninteractive From 2caf2a65bc1269a7fb4f512ccf2b09543089eab5 Mon Sep 17 00:00:00 2001 From: Manvenddra Rawat Date: Sat, 15 Apr 2023 17:43:07 +0530 Subject: [PATCH 4/6] updating local variable instead of env.USE_LINARO within Jenkinsfile Signed-off-by: Manvenddra Rawat --- Jenkins/Jenkinsfile | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/Jenkins/Jenkinsfile b/Jenkins/Jenkinsfile index 36df39d8b76..40c06a56f13 100755 --- a/Jenkins/Jenkinsfile +++ b/Jenkins/Jenkinsfile @@ -67,6 +67,7 @@ pipeline { echo 'Running commit checks...' script{ env.REPO_OR_FORK_URL = "${REPO_URL}" + using_linaro=env.USE_LINARO if (env.CHANGE_FORK != null) { env.REPO_OR_FORK_URL = "https://github.com/${env.CHANGE_FORK}/aimet" } @@ -74,15 +75,15 @@ pipeline { if ("${env.QCInternalValidation}" == "true") { if (didDockerFileChange("tf-gpu")) { // setting USE LINARO value to EMPTY to rebuild docker image - env.USE_LINARO="" + using_linaro="" } withCredentials([usernameColonPassword(credentialsId: 'neoci-pat', variable: 'USERPASS')]) { - sh "AIMET_VARIANT=\"tf-gpu\" bash -l -c \"cd ${REPO_NAME} && ${env.DEPENDENCY_DATA_PATH}/commit_check/commit-checks.sh -e AIMET_VARIANT ${env.USE_LINARO} ${env.PREBUILT_DOCKER_IMAGE_URL}\"" + sh "AIMET_VARIANT=\"tf-gpu\" bash -l -c \"cd ${REPO_NAME} && ${env.DEPENDENCY_DATA_PATH}/commit_check/commit-checks.sh -e AIMET_VARIANT ${using_linaro} ${env.PREBUILT_DOCKER_IMAGE_URL}\"" } } else { // setting USE LINARO value to EMPTY to rebuild docker image - env.USE_LINARO="" - sh "AIMET_VARIANT=\"tf-gpu\" bash -l -c \"cd ${REPO_NAME} && ${env.DEPENDENCY_DATA_PATH}/commit_check/commit-checks.sh -e AIMET_VARIANT ${env.USE_LINARO} ${env.PREBUILT_DOCKER_IMAGE_URL}\"" + using_linaro="" + sh "AIMET_VARIANT=\"tf-gpu\" bash -l -c \"cd ${REPO_NAME} && ${env.DEPENDENCY_DATA_PATH}/commit_check/commit-checks.sh -e AIMET_VARIANT ${using_linaro} ${env.PREBUILT_DOCKER_IMAGE_URL}\"" } } sh "bash -l -c \"rm -rf commit_checks_repo\"" @@ -633,7 +634,7 @@ def didDockerFileChange(aimet_variant) { def runStage(aimet_variant, options) { echo "*** Running stage ${options} for ${aimet_variant} variant on ${env.NODE_NAME} in workspace ${env.WORKSPACE_ROOT} ***" - + using_linaro=env.USE_LINARO if (aimet_variant == "default") { sh """ bash -l -c "cd ${REPO_NAME} && ./buildntest.sh ${options}" @@ -641,28 +642,29 @@ def runStage(aimet_variant, options) { } else { - if (didDockerFileChange(env.aimet_variant)) { + if (didDockerFileChange(aimet_variant)) { // setting USE LINARO value to EMPTY to rebuild docker image - env.USE_LINARO="" + using_linaro="" } sh """ - AIMET_VARIANT=${aimet_variant} bash -l -c "cd ${REPO_NAME} && ./buildntest.sh -e AIMET_VARIANT ${options} ${env.USE_LINARO} ${env.PREBUILT_DOCKER_IMAGE_URL}" + AIMET_VARIANT=${aimet_variant} bash -l -c "cd ${REPO_NAME} && ./buildntest.sh -e AIMET_VARIANT ${options} ${using_linaro} ${env.PREBUILT_DOCKER_IMAGE_URL}" """ } } def callAimetExtra(target_branch) { echo target_branch + using_linaro=env.USE_LINARO if (didDockerFileChange("tf-gpu") || didDockerFileChange("tf-cpu") || didDockerFileChange("torch-gpu") || didDockerFileChange("torch-cpu")) { // setting USE LINARO value to EMPTY to rebuild docker image - env.USE_LINARO="" + using_linaro="" } if (target_branch != "develop") { echo "Running AIMET additional stages on ${CHANGE_TARGET} branch ..." - build job: "AIMET-Extra", parameters: [string(name: 'AIMET_GIT_COMMIT', value: "${CHANGE_BRANCH}"), string(name: 'PROJECT_BRANCH', value: target_branch), string(name: 'USE_LINARO', value: "${env.USE_LINARO}")] + build job: "AIMET-Extra", parameters: [string(name: 'AIMET_GIT_COMMIT', value: "${CHANGE_BRANCH}"), string(name: 'PROJECT_BRANCH', value: target_branch), string(name: 'USE_LINARO', value: "${using_linaro}")] } else { echo "Running AIMET additional stages on develop branch ..." - build job: "AIMET-Extra", parameters: [string(name: 'AIMET_GIT_COMMIT', value: "${CHANGE_BRANCH}"), string(name: 'USE_LINARO', value: "${env.USE_LINARO}")] + build job: "AIMET-Extra", parameters: [string(name: 'AIMET_GIT_COMMIT', value: "${CHANGE_BRANCH}"), string(name: 'USE_LINARO', value: "${using_linaro}")] } } From 915b1345b5757e5216131743b968b53c02a4f579 Mon Sep 17 00:00:00 2001 From: Manvenddra Rawat Date: Sun, 16 Apr 2023 23:06:28 +0530 Subject: [PATCH 5/6] Moved linaro check outside of QCInternalValidation check Signed-off-by: Manvenddra Rawat --- Jenkins/Jenkinsfile | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Jenkins/Jenkinsfile b/Jenkins/Jenkinsfile index 40c06a56f13..f81c61f1154 100755 --- a/Jenkins/Jenkinsfile +++ b/Jenkins/Jenkinsfile @@ -72,17 +72,15 @@ pipeline { env.REPO_OR_FORK_URL = "https://github.com/${env.CHANGE_FORK}/aimet" } sh "bash -l -c \"git clone ${env.REPO_OR_FORK_URL} -b ${env.CHANGE_BRANCH} commit_checks_repo\"" - if ("${env.QCInternalValidation}" == "true") { - if (didDockerFileChange("tf-gpu")) { + if (didDockerFileChange("tf-gpu")) { // setting USE LINARO value to EMPTY to rebuild docker image using_linaro="" - } + } + if ("${env.QCInternalValidation}" == "true") { withCredentials([usernameColonPassword(credentialsId: 'neoci-pat', variable: 'USERPASS')]) { sh "AIMET_VARIANT=\"tf-gpu\" bash -l -c \"cd ${REPO_NAME} && ${env.DEPENDENCY_DATA_PATH}/commit_check/commit-checks.sh -e AIMET_VARIANT ${using_linaro} ${env.PREBUILT_DOCKER_IMAGE_URL}\"" } } else { - // setting USE LINARO value to EMPTY to rebuild docker image - using_linaro="" sh "AIMET_VARIANT=\"tf-gpu\" bash -l -c \"cd ${REPO_NAME} && ${env.DEPENDENCY_DATA_PATH}/commit_check/commit-checks.sh -e AIMET_VARIANT ${using_linaro} ${env.PREBUILT_DOCKER_IMAGE_URL}\"" } } From a6632be411bd589bb51c3580f9b15c446ff4cc00 Mon Sep 17 00:00:00 2001 From: Manvenddra Rawat Date: Mon, 17 Apr 2023 13:30:59 +0530 Subject: [PATCH 6/6] Fixed typo with BUILD_TIMEOUT Signed-off-by: Manvenddra Rawat --- Jenkins/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Jenkins/Jenkinsfile b/Jenkins/Jenkinsfile index f81c61f1154..032c43a2916 100755 --- a/Jenkins/Jenkinsfile +++ b/Jenkins/Jenkinsfile @@ -10,7 +10,7 @@ pipeline { agent { label "${params.BUILD_LABEL_CPU}" } options { timestamps() - timeout( time: "${BUILD_TIMEOUT}", unit: 'HOURS' ) + timeout( time: "${params.BUILD_TIMEOUT}", unit: 'HOURS' ) } environment{ def workspace = pwd()