From 5a767d053ddca688bb0e2e463106ae4c5d289b46 Mon Sep 17 00:00:00 2001 From: Florin Blanaru Date: Mon, 14 Nov 2022 18:32:12 +0100 Subject: [PATCH] [CI] Separate the ci scripts into Github and Jenkins scripts (#13368) This PR is a duplicate of #12940 and #12941. For some reason, I am unable to reopen #12940. --- .github/disabled_workflows/pr_comment_bot.yml | 4 +- .github/workflows/cc_bot.yml | 2 +- .github/workflows/nightly_docker_update.yml | 2 +- .github/workflows/ping_reviewers.yml | 2 +- .github/workflows/tag_teams.yml | 2 +- .github/workflows/tvmbot.yml | 2 +- .../update_last_successful_branch.yml | 2 +- Jenkinsfile | 176 +++++++++--------- ci/jenkins/Deploy.groovy.j2 | 4 +- ci/jenkins/DockerBuild.groovy.j2 | 4 +- ci/jenkins/Jenkinsfile.j2 | 2 + ci/jenkins/Prepare.groovy.j2 | 20 +- ci/jenkins/macros.j2 | 4 +- ci/scripts/{ => github}/__init__.py | 2 +- .../{ => github}/github_cc_reviewers.py | 4 + ci/scripts/{ => github}/github_commenter.py | 6 + .../{ => github}/github_docs_comment.py | 0 ci/scripts/{ => github}/github_pr_comment.py | 6 + .../github_skipped_tests_comment.py | 2 +- ci/scripts/{ => github}/github_tag_teams.py | 5 + ci/scripts/{ => github}/github_tvmbot.py | 5 + ci/scripts/{ => github}/ping_reviewers.py | 11 +- ci/scripts/{ => github}/update_branch.py | 7 +- ci/scripts/{ => jenkins}/check_pr.py | 0 ci/scripts/{ => jenkins}/cmd_utils.py | 3 +- .../{ => jenkins}/determine_docker_images.py | 3 +- ci/scripts/{ => jenkins}/git_change_docker.sh | 0 ci/scripts/{ => jenkins}/git_change_docs.sh | 0 ci/scripts/{ => jenkins}/git_skip_ci.py | 0 ci/scripts/{ => jenkins}/git_skip_ci_globs.py | 0 ci/scripts/{ => jenkins}/git_utils.py | 0 ci/scripts/{ => jenkins}/http_utils.py | 0 .../{ => jenkins}/open_docker_update_pr.py | 3 +- ci/scripts/{ => jenkins}/pytest_ids.py | 0 ci/scripts/{ => jenkins}/pytest_wrapper.py | 3 +- ci/scripts/{ => jenkins}/retry.sh | 0 .../{ => jenkins}/should_rebuild_docker.py | 0 .../{ => jenkins}/should_run_slow_tests.py | 6 +- ci/scripts/required_tests_to_run.json | 11 -- docker/bash.sh | 3 +- tests/python/ci/test_ci.py | 38 ++-- tests/python/ci/test_tvmbot.py | 4 +- tests/python/ci/test_utils.py | 2 + tests/scripts/release/gather_prs.py | 3 +- tests/scripts/release/make_notes.py | 2 + tests/scripts/setup-pytest-env.sh | 2 +- tests/scripts/task_build.py | 4 +- tests/scripts/task_python_frontend.sh | 2 +- 48 files changed, 197 insertions(+), 166 deletions(-) rename ci/scripts/{ => github}/__init__.py (94%) rename ci/scripts/{ => github}/github_cc_reviewers.py (95%) rename ci/scripts/{ => github}/github_commenter.py (95%) rename ci/scripts/{ => github}/github_docs_comment.py (100%) rename ci/scripts/{ => github}/github_pr_comment.py (94%) rename ci/scripts/{ => github}/github_skipped_tests_comment.py (99%) rename ci/scripts/{ => github}/github_tag_teams.py (97%) rename ci/scripts/{ => github}/github_tvmbot.py (99%) rename ci/scripts/{ => github}/ping_reviewers.py (96%) rename ci/scripts/{ => github}/update_branch.py (96%) rename ci/scripts/{ => jenkins}/check_pr.py (100%) rename ci/scripts/{ => jenkins}/cmd_utils.py (95%) rename ci/scripts/{ => jenkins}/determine_docker_images.py (98%) rename ci/scripts/{ => jenkins}/git_change_docker.sh (100%) rename ci/scripts/{ => jenkins}/git_change_docs.sh (100%) rename ci/scripts/{ => jenkins}/git_skip_ci.py (100%) rename ci/scripts/{ => jenkins}/git_skip_ci_globs.py (100%) rename ci/scripts/{ => jenkins}/git_utils.py (100%) rename ci/scripts/{ => jenkins}/http_utils.py (100%) rename ci/scripts/{ => jenkins}/open_docker_update_pr.py (99%) rename ci/scripts/{ => jenkins}/pytest_ids.py (100%) rename ci/scripts/{ => jenkins}/pytest_wrapper.py (98%) rename ci/scripts/{ => jenkins}/retry.sh (100%) rename ci/scripts/{ => jenkins}/should_rebuild_docker.py (100%) rename ci/scripts/{ => jenkins}/should_run_slow_tests.py (95%) delete mode 100644 ci/scripts/required_tests_to_run.json diff --git a/.github/disabled_workflows/pr_comment_bot.yml b/.github/disabled_workflows/pr_comment_bot.yml index 1ac33c77d2e5..436286768502 100644 --- a/.github/disabled_workflows/pr_comment_bot.yml +++ b/.github/disabled_workflows/pr_comment_bot.yml @@ -38,7 +38,7 @@ jobs: PR_NUMBER: ${{ github.event.number }} run: | set -eux - python ci/scripts/github_pr_comment.py --pr "$PR_NUMBER" + python ci/scripts/github/github_pr_comment.py --pr "$PR_NUMBER" - name: Comment bot comment (status) if: ${{ github.event.state }} env: @@ -49,7 +49,7 @@ jobs: if [[ "$URL" == *"PR-"* ]]; then echo "PR status, sending comment" PR_NUMBER=$(echo $URL | sed 's/.*PR-//g' | sed 's/\/.*//g') - python ci/scripts/github_pr_comment.py --pr "$PR_NUMBER" || /bin/true + python ci/scripts/github/github_pr_comment.py --pr "$PR_NUMBER" || /bin/true else echo "Not a PR status, skipping" fi diff --git a/.github/workflows/cc_bot.yml b/.github/workflows/cc_bot.yml index 95aa96426229..54640148af73 100644 --- a/.github/workflows/cc_bot.yml +++ b/.github/workflows/cc_bot.yml @@ -44,4 +44,4 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set -eux - python ci/scripts/github_cc_reviewers.py || echo step failed + python ci/scripts/github/github_cc_reviewers.py || echo step failed diff --git a/.github/workflows/nightly_docker_update.yml b/.github/workflows/nightly_docker_update.yml index c2441807430f..350987487306 100644 --- a/.github/workflows/nightly_docker_update.yml +++ b/.github/workflows/nightly_docker_update.yml @@ -28,4 +28,4 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set -eux - python ci/scripts/open_docker_update_pr.py + python ci/scripts/jenkins/open_docker_update_pr.py diff --git a/.github/workflows/ping_reviewers.yml b/.github/workflows/ping_reviewers.yml index a2e3e996a033..f0395c09474b 100644 --- a/.github/workflows/ping_reviewers.yml +++ b/.github/workflows/ping_reviewers.yml @@ -20,4 +20,4 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set -eux - python ci/scripts/ping_reviewers.py --wait-time-minutes 10080 || echo failed + python ci/scripts/github/ping_reviewers.py --wait-time-minutes 10080 || echo failed diff --git a/.github/workflows/tag_teams.yml b/.github/workflows/tag_teams.yml index c0c1b8b8299d..14d29604767e 100644 --- a/.github/workflows/tag_teams.yml +++ b/.github/workflows/tag_teams.yml @@ -38,4 +38,4 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set -eux - python ci/scripts/github_tag_teams.py || echo failed + python ci/scripts/github/github_tag_teams.py || echo failed diff --git a/.github/workflows/tvmbot.yml b/.github/workflows/tvmbot.yml index 23e90aed5329..6965ea86f2e1 100644 --- a/.github/workflows/tvmbot.yml +++ b/.github/workflows/tvmbot.yml @@ -34,4 +34,4 @@ jobs: RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} run: | set -eux - python ci/scripts/github_tvmbot.py --pr "$PR_NUMBER" --run-url "$RUN_URL" --trigger-comment-json "$ISSUE_COMMENT" + python ci/scripts/github/github_tvmbot.py --pr "$PR_NUMBER" --run-url "$RUN_URL" --trigger-comment-json "$ISSUE_COMMENT" diff --git a/.github/workflows/update_last_successful_branch.yml b/.github/workflows/update_last_successful_branch.yml index 6635b9ef4c47..da8852d8d296 100644 --- a/.github/workflows/update_last_successful_branch.yml +++ b/.github/workflows/update_last_successful_branch.yml @@ -41,4 +41,4 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | set -eux - python ci/scripts/update_branch.py || echo step failed + python ci/scripts/github/update_branch.py || echo step failed diff --git a/Jenkinsfile b/Jenkinsfile index 135f64dc1d94..171e200df379 100755 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -106,6 +106,8 @@ rebuild_docker_images = false // Filenames for stashing between build and test steps s3_prefix = "tvm-jenkins-artifacts-prod/tvm/${env.BRANCH_NAME}/${env.BUILD_NUMBER}" +// Jenkins script root directory +jenkins_scripts_root = "ci/scripts/jenkins" // General note: Jenkins has limits on the size of a method (or top level code) // that are pretty strict, so most usage of groovy methods in these templates @@ -146,7 +148,7 @@ def init_git() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 timeout 5m git submodule update --init -f --jobs 0 """, label: 'Update git submodules', @@ -178,7 +180,7 @@ def docker_init(image) { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 5 docker pull ${image} """, label: 'Pull docker image', @@ -194,7 +196,7 @@ def should_skip_slow_tests(pr_number) { // Exit code of 1 means run slow tests, exit code of 0 means skip slow tests result = sh ( returnStatus: true, - script: "./ci/scripts/should_run_slow_tests.py --pr '${pr_number}'", + script: "./${jenkins_scripts_root}/should_run_slow_tests.py --pr '${pr_number}'", label: 'Check if CI should run slow tests', ) } @@ -230,7 +232,7 @@ def checkout_trusted_files() { // (especially those that access secrets) should be checked out here so // only trusted versions are used in CI sh( - script: "git checkout ${upstream_revision} ci/scripts/.", + script: "git checkout ${upstream_revision} ${jenkins_scripts_root}/.", label: 'Check out trusted files', ) } @@ -243,7 +245,7 @@ def should_skip_ci(pr_number) { } glob_skip_ci_code = sh ( returnStatus: true, - script: "./ci/scripts/git_skip_ci_globs.py", + script: "./${jenkins_scripts_root}/git_skip_ci_globs.py", label: 'Check if CI should be skipped due to changed files', ) if (glob_skip_ci_code == 0) { @@ -257,7 +259,7 @@ def should_skip_ci(pr_number) { // full CI just in case). Exit code of 0 means skip CI. git_skip_ci_code = sh ( returnStatus: true, - script: "./ci/scripts/git_skip_ci.py --pr '${pr_number}'", + script: "./${jenkins_scripts_root}/git_skip_ci.py --pr '${pr_number}'", label: 'Check if CI should be skipped', ) } @@ -274,7 +276,7 @@ def check_pr(pr_number) { variable: 'GITHUB_TOKEN', )]) { sh ( - script: "python3 ci/scripts/check_pr.py --pr ${pr_number}", + script: "python3 ${jenkins_scripts_root}/check_pr.py --pr ${pr_number}", label: 'Check PR title and body', ) } @@ -291,7 +293,7 @@ def prepare() { if (env.DETERMINE_DOCKER_IMAGES == 'yes') { sh( - script: "./ci/scripts/determine_docker_images.py ci_arm=${ci_arm} ci_cortexm=${ci_cortexm} ci_cpu=${ci_cpu} ci_gpu=${ci_gpu} ci_hexagon=${ci_hexagon} ci_i386=${ci_i386} ci_lint=${ci_lint} ci_minimal=${ci_minimal} ci_riscv=${ci_riscv} ci_wasm=${ci_wasm} ", + script: "./${jenkins_scripts_root}/determine_docker_images.py ci_arm=${ci_arm} ci_cortexm=${ci_cortexm} ci_cpu=${ci_cpu} ci_gpu=${ci_gpu} ci_hexagon=${ci_hexagon} ci_i386=${ci_i386} ci_lint=${ci_lint} ci_minimal=${ci_minimal} ci_riscv=${ci_riscv} ci_wasm=${ci_wasm} ", label: 'Decide whether to use tlcpack or tlcpackstaging for Docker images', ) // Pull image names from the results of should_rebuild_docker.py @@ -374,14 +376,14 @@ def prepare() { is_docs_only_build = sh ( returnStatus: true, - script: './ci/scripts/git_change_docs.sh', + script: './${jenkins_scripts_root}/git_change_docs.sh', label: 'Check for docs only changes', ) skip_ci = should_skip_ci(env.CHANGE_ID) skip_slow_tests = should_skip_slow_tests(env.CHANGE_ID) rebuild_docker_images = sh ( returnStatus: true, - script: './ci/scripts/git_change_docker.sh', + script: './${jenkins_scripts_root}/git_change_docker.sh', label: 'Check for any docker changes', ) @@ -416,7 +418,7 @@ def ecr_push(full_name) { sh( script: """ set -x - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh docker tag ${full_name} \$AWS_ECR_REPO/${full_name} retry 5 docker push \$AWS_ECR_REPO/${full_name} """, @@ -459,7 +461,7 @@ def ecr_pull(full_name) { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 5 docker pull ${full_name} """, label: 'Pull image from ECR' @@ -745,7 +747,7 @@ stage('Build') { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh md5sum build/libtvm.so retry 3 aws s3 cp --no-progress build/libtvm.so s3://${s3_prefix}/gpu/build/libtvm.so md5sum build/libvta_fsim.so @@ -766,7 +768,7 @@ stage('Build') { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh md5sum build/libtvm.so retry 3 aws s3 cp --no-progress build/libtvm.so s3://${s3_prefix}/gpu2/build/libtvm.so md5sum build/libvta_fsim.so @@ -801,7 +803,7 @@ stage('Build') { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh md5sum build/libvta_tsim.so retry 3 aws s3 cp --no-progress build/libvta_tsim.so s3://${s3_prefix}/cpu/build/libvta_tsim.so md5sum build/libtvm.so @@ -843,7 +845,7 @@ stage('Build') { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh md5sum build/libtvm.so retry 3 aws s3 cp --no-progress build/libtvm.so s3://${s3_prefix}/cpu-minimal/build/libtvm.so md5sum build/libtvm_runtime.so @@ -902,7 +904,7 @@ stage('Build') { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh md5sum build/libvta_tsim.so retry 3 aws s3 cp --no-progress build/libvta_tsim.so s3://${s3_prefix}/i386/build/libvta_tsim.so md5sum build/libtvm.so @@ -939,7 +941,7 @@ stage('Build') { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh md5sum build/libtvm.so retry 3 aws s3 cp --no-progress build/libtvm.so s3://${s3_prefix}/arm/build/libtvm.so md5sum build/libvta_fsim.so @@ -974,7 +976,7 @@ stage('Build') { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh md5sum build/libtvm.so retry 3 aws s3 cp --no-progress build/libtvm.so s3://${s3_prefix}/cortexm/build/libtvm.so md5sum build/libtvm_runtime.so @@ -1012,7 +1014,7 @@ stage('Build') { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh md5sum build/libtvm.so retry 3 aws s3 cp --no-progress build/libtvm.so s3://${s3_prefix}/hexagon/build/libtvm.so md5sum build/libtvm_runtime.so @@ -1046,7 +1048,7 @@ stage('Build') { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh md5sum build/libtvm.so retry 3 aws s3 cp --no-progress build/libtvm.so s3://${s3_prefix}/riscv/build/libtvm.so md5sum build/libtvm_runtime.so @@ -1091,7 +1093,7 @@ def shard_run_unittest_GPU_1_of_3() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu2/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu2/build/libvta_fsim.so build/libvta_fsim.so @@ -1109,7 +1111,7 @@ def shard_run_unittest_GPU_1_of_3() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so @@ -1174,7 +1176,7 @@ def shard_run_unittest_GPU_2_of_3() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so @@ -1242,7 +1244,7 @@ def shard_run_unittest_GPU_3_of_3() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so @@ -1307,7 +1309,7 @@ def shard_run_integration_CPU_1_of_4() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_tsim.so build/libvta_tsim.so md5sum build/libvta_tsim.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm.so build/libtvm.so @@ -1369,7 +1371,7 @@ def shard_run_integration_CPU_2_of_4() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_tsim.so build/libvta_tsim.so md5sum build/libvta_tsim.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm.so build/libtvm.so @@ -1431,7 +1433,7 @@ def shard_run_integration_CPU_3_of_4() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_tsim.so build/libvta_tsim.so md5sum build/libvta_tsim.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm.so build/libtvm.so @@ -1493,7 +1495,7 @@ def shard_run_integration_CPU_4_of_4() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_tsim.so build/libvta_tsim.so md5sum build/libvta_tsim.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm.so build/libtvm.so @@ -1556,7 +1558,7 @@ def shard_run_python_i386_1_of_3() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libvta_fsim.so build/libvta_fsim.so @@ -1618,7 +1620,7 @@ def shard_run_python_i386_2_of_3() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libvta_fsim.so build/libvta_fsim.so @@ -1680,7 +1682,7 @@ def shard_run_python_i386_3_of_3() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libvta_fsim.so build/libvta_fsim.so @@ -1742,7 +1744,7 @@ def shard_run_test_Hexagon_1_of_8() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm_runtime.so build/libtvm_runtime.so @@ -1803,7 +1805,7 @@ def shard_run_test_Hexagon_2_of_8() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm_runtime.so build/libtvm_runtime.so @@ -1863,7 +1865,7 @@ def shard_run_test_Hexagon_3_of_8() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm_runtime.so build/libtvm_runtime.so @@ -1923,7 +1925,7 @@ def shard_run_test_Hexagon_4_of_8() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm_runtime.so build/libtvm_runtime.so @@ -1983,7 +1985,7 @@ def shard_run_test_Hexagon_5_of_8() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm_runtime.so build/libtvm_runtime.so @@ -2043,7 +2045,7 @@ def shard_run_test_Hexagon_6_of_8() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm_runtime.so build/libtvm_runtime.so @@ -2103,7 +2105,7 @@ def shard_run_test_Hexagon_7_of_8() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm_runtime.so build/libtvm_runtime.so @@ -2163,7 +2165,7 @@ def shard_run_test_Hexagon_8_of_8() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm_runtime.so build/libtvm_runtime.so @@ -2224,7 +2226,7 @@ def shard_run_integration_aarch64_1_of_4() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libvta_fsim.so build/libvta_fsim.so @@ -2285,7 +2287,7 @@ def shard_run_integration_aarch64_2_of_4() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libvta_fsim.so build/libvta_fsim.so @@ -2346,7 +2348,7 @@ def shard_run_integration_aarch64_3_of_4() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libvta_fsim.so build/libvta_fsim.so @@ -2407,7 +2409,7 @@ def shard_run_integration_aarch64_4_of_4() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libvta_fsim.so build/libvta_fsim.so @@ -2469,7 +2471,7 @@ def shard_run_topi_GPU_1_of_3() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so @@ -2529,7 +2531,7 @@ def shard_run_topi_GPU_2_of_3() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so @@ -2589,7 +2591,7 @@ def shard_run_topi_GPU_3_of_3() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so @@ -2650,7 +2652,7 @@ def shard_run_frontend_GPU_1_of_6() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so @@ -2710,7 +2712,7 @@ def shard_run_frontend_GPU_2_of_6() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so @@ -2770,7 +2772,7 @@ def shard_run_frontend_GPU_3_of_6() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so @@ -2830,7 +2832,7 @@ def shard_run_frontend_GPU_4_of_6() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so @@ -2890,7 +2892,7 @@ def shard_run_frontend_GPU_5_of_6() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so @@ -2950,7 +2952,7 @@ def shard_run_frontend_GPU_6_of_6() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so @@ -3011,7 +3013,7 @@ def shard_run_topi_aarch64_1_of_2() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libvta_fsim.so build/libvta_fsim.so @@ -3076,7 +3078,7 @@ def shard_run_topi_aarch64_2_of_2() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libvta_fsim.so build/libvta_fsim.so @@ -3141,7 +3143,7 @@ def shard_run_frontend_aarch64_1_of_2() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libvta_fsim.so build/libvta_fsim.so @@ -3201,7 +3203,7 @@ def shard_run_frontend_aarch64_2_of_2() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libvta_fsim.so build/libvta_fsim.so @@ -3262,7 +3264,7 @@ def shard_run_test_Cortex_M_1_of_12() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cortexm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cortexm/build/libtvm_runtime.so build/libtvm_runtime.so @@ -3327,7 +3329,7 @@ def shard_run_test_Cortex_M_2_of_12() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cortexm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cortexm/build/libtvm_runtime.so build/libtvm_runtime.so @@ -3387,7 +3389,7 @@ def shard_run_test_Cortex_M_3_of_12() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cortexm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cortexm/build/libtvm_runtime.so build/libtvm_runtime.so @@ -3447,7 +3449,7 @@ def shard_run_test_Cortex_M_4_of_12() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cortexm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cortexm/build/libtvm_runtime.so build/libtvm_runtime.so @@ -3507,7 +3509,7 @@ def shard_run_test_Cortex_M_5_of_12() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cortexm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cortexm/build/libtvm_runtime.so build/libtvm_runtime.so @@ -3567,7 +3569,7 @@ def shard_run_test_Cortex_M_6_of_12() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cortexm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cortexm/build/libtvm_runtime.so build/libtvm_runtime.so @@ -3627,7 +3629,7 @@ def shard_run_test_Cortex_M_7_of_12() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cortexm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cortexm/build/libtvm_runtime.so build/libtvm_runtime.so @@ -3687,7 +3689,7 @@ def shard_run_test_Cortex_M_8_of_12() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cortexm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cortexm/build/libtvm_runtime.so build/libtvm_runtime.so @@ -3747,7 +3749,7 @@ def shard_run_test_Cortex_M_9_of_12() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cortexm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cortexm/build/libtvm_runtime.so build/libtvm_runtime.so @@ -3807,7 +3809,7 @@ def shard_run_test_Cortex_M_10_of_12() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cortexm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cortexm/build/libtvm_runtime.so build/libtvm_runtime.so @@ -3867,7 +3869,7 @@ def shard_run_test_Cortex_M_11_of_12() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cortexm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cortexm/build/libtvm_runtime.so build/libtvm_runtime.so @@ -3927,7 +3929,7 @@ def shard_run_test_Cortex_M_12_of_12() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cortexm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cortexm/build/libtvm_runtime.so build/libtvm_runtime.so @@ -3988,7 +3990,7 @@ def shard_run_test_RISC_V_1_of_1() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/riscv/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/riscv/build/libtvm_runtime.so build/libtvm_runtime.so @@ -4045,7 +4047,7 @@ def run_unittest_minimal() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu-minimal/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu-minimal/build/libtvm_runtime.so build/libtvm_runtime.so @@ -4250,7 +4252,7 @@ stage('Test') { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_tsim.so build/libvta_tsim.so md5sum build/libvta_tsim.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm.so build/libtvm.so @@ -4311,7 +4313,7 @@ stage('Test') { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_fsim.so build/libvta_fsim.so @@ -4362,7 +4364,7 @@ stage('Test') { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so retry 3 aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so @@ -4387,7 +4389,7 @@ stage('Test') { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh md5sum docs.tgz retry 3 aws s3 cp --no-progress docs.tgz s3://${s3_prefix}/docs/docs.tgz """, @@ -4437,7 +4439,7 @@ def update_docker(ecr_image, hub_image) { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh docker tag \ ${ecr_image} \ ${hub_image} @@ -4501,7 +4503,7 @@ def deploy() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 aws s3 cp --no-progress s3://${s3_prefix}/docs/docs.tgz docs.tgz md5sum docs.tgz """, @@ -4582,7 +4584,7 @@ def deploy() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh docker pull tlcpackstaging/ci_arm:${tag} docker tag tlcpackstaging/ci_arm:${tag} tlcpack/ci-arm:${tag} retry 5 docker push tlcpack/ci-arm:${tag} @@ -4596,7 +4598,7 @@ def deploy() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh docker pull tlcpackstaging/ci_cortexm:${tag} docker tag tlcpackstaging/ci_cortexm:${tag} tlcpack/ci-cortexm:${tag} retry 5 docker push tlcpack/ci-cortexm:${tag} @@ -4610,7 +4612,7 @@ def deploy() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh docker pull tlcpackstaging/ci_cpu:${tag} docker tag tlcpackstaging/ci_cpu:${tag} tlcpack/ci-cpu:${tag} retry 5 docker push tlcpack/ci-cpu:${tag} @@ -4624,7 +4626,7 @@ def deploy() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh docker pull tlcpackstaging/ci_gpu:${tag} docker tag tlcpackstaging/ci_gpu:${tag} tlcpack/ci-gpu:${tag} retry 5 docker push tlcpack/ci-gpu:${tag} @@ -4638,7 +4640,7 @@ def deploy() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh docker pull tlcpackstaging/ci_hexagon:${tag} docker tag tlcpackstaging/ci_hexagon:${tag} tlcpack/ci-hexagon:${tag} retry 5 docker push tlcpack/ci-hexagon:${tag} @@ -4652,7 +4654,7 @@ def deploy() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh docker pull tlcpackstaging/ci_i386:${tag} docker tag tlcpackstaging/ci_i386:${tag} tlcpack/ci-i386:${tag} retry 5 docker push tlcpack/ci-i386:${tag} @@ -4666,7 +4668,7 @@ def deploy() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh docker pull tlcpackstaging/ci_lint:${tag} docker tag tlcpackstaging/ci_lint:${tag} tlcpack/ci-lint:${tag} retry 5 docker push tlcpack/ci-lint:${tag} @@ -4680,7 +4682,7 @@ def deploy() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh docker pull tlcpackstaging/ci_minimal:${tag} docker tag tlcpackstaging/ci_minimal:${tag} tlcpack/ci-minimal:${tag} retry 5 docker push tlcpack/ci-minimal:${tag} @@ -4694,7 +4696,7 @@ def deploy() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh docker pull tlcpackstaging/ci_riscv:${tag} docker tag tlcpackstaging/ci_riscv:${tag} tlcpack/ci-riscv:${tag} retry 5 docker push tlcpack/ci-riscv:${tag} @@ -4708,7 +4710,7 @@ def deploy() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh docker pull tlcpackstaging/ci_wasm:${tag} docker tag tlcpackstaging/ci_wasm:${tag} tlcpack/ci-wasm:${tag} retry 5 docker push tlcpack/ci-wasm:${tag} diff --git a/ci/jenkins/Deploy.groovy.j2 b/ci/jenkins/Deploy.groovy.j2 index 798af6736e1e..f11d901258f6 100644 --- a/ci/jenkins/Deploy.groovy.j2 +++ b/ci/jenkins/Deploy.groovy.j2 @@ -30,7 +30,7 @@ def update_docker(ecr_image, hub_image) { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh docker tag \ ${ecr_image} \ ${hub_image} @@ -148,7 +148,7 @@ def deploy() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh docker pull tlcpackstaging/{{ image.name }}:${tag} docker tag tlcpackstaging/{{ image.name }}:${tag} tlcpack/{{ image.name.replace("_", "-") }}:${tag} retry 5 docker push tlcpack/{{ image.name.replace("_", "-") }}:${tag} diff --git a/ci/jenkins/DockerBuild.groovy.j2 b/ci/jenkins/DockerBuild.groovy.j2 index 5ffbeded80fa..69e0db4f9e4f 100644 --- a/ci/jenkins/DockerBuild.groovy.j2 +++ b/ci/jenkins/DockerBuild.groovy.j2 @@ -21,7 +21,7 @@ def ecr_push(full_name) { sh( script: """ set -x - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh docker tag ${full_name} \$AWS_ECR_REPO/${full_name} retry 5 docker push \$AWS_ECR_REPO/${full_name} """, @@ -64,7 +64,7 @@ def ecr_pull(full_name) { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 5 docker pull ${full_name} """, label: 'Pull image from ECR' diff --git a/ci/jenkins/Jenkinsfile.j2 b/ci/jenkins/Jenkinsfile.j2 index 34c1d66e43ba..7ceef81e4e7c 100644 --- a/ci/jenkins/Jenkinsfile.j2 +++ b/ci/jenkins/Jenkinsfile.j2 @@ -100,6 +100,8 @@ rebuild_docker_images = false {% set hexagon_api = ['build/hexagon_api_output',] %} s3_prefix = "tvm-jenkins-artifacts-prod/tvm/${env.BRANCH_NAME}/${env.BUILD_NUMBER}" +// Jenkins script root directory +jenkins_scripts_root = "ci/scripts/jenkins" {% set aws_default_region = "us-west-2" %} {% set aws_ecr_url = "dkr.ecr." + aws_default_region + ".amazonaws.com" %} diff --git a/ci/jenkins/Prepare.groovy.j2 b/ci/jenkins/Prepare.groovy.j2 index 2c6f93090127..8bc57f466225 100644 --- a/ci/jenkins/Prepare.groovy.j2 +++ b/ci/jenkins/Prepare.groovy.j2 @@ -34,7 +34,7 @@ def init_git() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 3 timeout 5m git submodule update --init -f --jobs 0 """, label: 'Update git submodules', @@ -66,7 +66,7 @@ def docker_init(image) { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh retry 5 docker pull ${image} """, label: 'Pull docker image', @@ -82,7 +82,7 @@ def should_skip_slow_tests(pr_number) { // Exit code of 1 means run slow tests, exit code of 0 means skip slow tests result = sh ( returnStatus: true, - script: "./ci/scripts/should_run_slow_tests.py --pr '${pr_number}'", + script: "./${jenkins_scripts_root}/should_run_slow_tests.py --pr '${pr_number}'", label: 'Check if CI should run slow tests', ) } @@ -118,7 +118,7 @@ def checkout_trusted_files() { // (especially those that access secrets) should be checked out here so // only trusted versions are used in CI sh( - script: "git checkout ${upstream_revision} ci/scripts/.", + script: "git checkout ${upstream_revision} ${jenkins_scripts_root}/.", label: 'Check out trusted files', ) } @@ -131,7 +131,7 @@ def should_skip_ci(pr_number) { } glob_skip_ci_code = sh ( returnStatus: true, - script: "./ci/scripts/git_skip_ci_globs.py", + script: "./${jenkins_scripts_root}/git_skip_ci_globs.py", label: 'Check if CI should be skipped due to changed files', ) if (glob_skip_ci_code == 0) { @@ -145,7 +145,7 @@ def should_skip_ci(pr_number) { // full CI just in case). Exit code of 0 means skip CI. git_skip_ci_code = sh ( returnStatus: true, - script: "./ci/scripts/git_skip_ci.py --pr '${pr_number}'", + script: "./${jenkins_scripts_root}/git_skip_ci.py --pr '${pr_number}'", label: 'Check if CI should be skipped', ) } @@ -162,7 +162,7 @@ def check_pr(pr_number) { variable: 'GITHUB_TOKEN', )]) { sh ( - script: "python3 ci/scripts/check_pr.py --pr ${pr_number}", + script: "python3 ${jenkins_scripts_root}/check_pr.py --pr ${pr_number}", label: 'Check PR title and body', ) } @@ -179,7 +179,7 @@ def prepare() { if (env.DETERMINE_DOCKER_IMAGES == 'yes') { sh( - script: "./ci/scripts/determine_docker_images.py {% for image in images %}{{ image.name }}={% raw %}${{% endraw %}{{ image.name }}{% raw %}}{% endraw %} {% endfor %}", + script: "./${jenkins_scripts_root}/determine_docker_images.py {% for image in images %}{{ image.name }}={% raw %}${{% endraw %}{{ image.name }}{% raw %}}{% endraw %} {% endfor %}", label: 'Decide whether to use tlcpack or tlcpackstaging for Docker images', ) // Pull image names from the results of should_rebuild_docker.py @@ -205,14 +205,14 @@ def prepare() { is_docs_only_build = sh ( returnStatus: true, - script: './ci/scripts/git_change_docs.sh', + script: './${jenkins_scripts_root}/git_change_docs.sh', label: 'Check for docs only changes', ) skip_ci = should_skip_ci(env.CHANGE_ID) skip_slow_tests = should_skip_slow_tests(env.CHANGE_ID) rebuild_docker_images = sh ( returnStatus: true, - script: './ci/scripts/git_change_docker.sh', + script: './${jenkins_scripts_root}/git_change_docker.sh', label: 'Check for any docker changes', ) diff --git a/ci/jenkins/macros.j2 b/ci/jenkins/macros.j2 index 78c5acd1c7ff..b8ac0de91ce6 100644 --- a/ci/jenkins/macros.j2 +++ b/ci/jenkins/macros.j2 @@ -191,7 +191,7 @@ def {{ method_name }}() { sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh {% for filename in filenames %} md5sum {{ filename }} retry 3 aws s3 cp --no-progress {{ filename }} s3://${s3_prefix}/{{ tag }}/{{ filename }} @@ -208,7 +208,7 @@ sh( sh( script: """ set -eux - . ci/scripts/retry.sh + . ${jenkins_scripts_root}/retry.sh {% for filename in filenames %} retry 3 aws s3 cp --no-progress s3://${s3_prefix}/{{ tag }}/{{ filename }} {{ filename }} md5sum {{ filename }} diff --git a/ci/scripts/__init__.py b/ci/scripts/github/__init__.py similarity index 94% rename from ci/scripts/__init__.py rename to ci/scripts/github/__init__.py index 064781fa158d..edc5fdff0e09 100644 --- a/ci/scripts/__init__.py +++ b/ci/scripts/github/__init__.py @@ -14,6 +14,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -"""Package to enable testing of CI scripts""" +"""Package to enable testing of Github scripts""" from . import github_skipped_tests_comment, github_pr_comment, github_tag_teams, github_docs_comment diff --git a/ci/scripts/github_cc_reviewers.py b/ci/scripts/github/github_cc_reviewers.py similarity index 95% rename from ci/scripts/github_cc_reviewers.py rename to ci/scripts/github/github_cc_reviewers.py index d8323221a7b0..b98a0290c17c 100755 --- a/ci/scripts/github_cc_reviewers.py +++ b/ci/scripts/github/github_cc_reviewers.py @@ -21,9 +21,13 @@ import json import argparse import re +from pathlib import Path from urllib import error from typing import Dict, Any, List +# Hackery to enable importing of utils from ci/scripts/jenkins +REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent +sys.path.append(str(REPO_ROOT / "ci" / "scripts" / "jenkins")) from git_utils import git, GitHubRepo, parse_remote diff --git a/ci/scripts/github_commenter.py b/ci/scripts/github/github_commenter.py similarity index 95% rename from ci/scripts/github_commenter.py rename to ci/scripts/github/github_commenter.py index dc71fcd1fd32..909ce8516402 100644 --- a/ci/scripts/github_commenter.py +++ b/ci/scripts/github/github_commenter.py @@ -18,8 +18,14 @@ import re import logging +import sys +from pathlib import Path from typing import Dict, Tuple, Any, Optional, List, Union +# Hackery to enable importing of utils from ci/scripts/jenkins +REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent +sys.path.append(str(REPO_ROOT / "ci" / "scripts" / "jenkins")) + from git_utils import GitHubRepo BOT_COMMENT_START = "" diff --git a/ci/scripts/github_docs_comment.py b/ci/scripts/github/github_docs_comment.py similarity index 100% rename from ci/scripts/github_docs_comment.py rename to ci/scripts/github/github_docs_comment.py diff --git a/ci/scripts/github_pr_comment.py b/ci/scripts/github/github_pr_comment.py similarity index 94% rename from ci/scripts/github_pr_comment.py rename to ci/scripts/github/github_pr_comment.py index bcf4c5096ab0..3ad7b02674c7 100755 --- a/ci/scripts/github_pr_comment.py +++ b/ci/scripts/github/github_pr_comment.py @@ -18,6 +18,12 @@ import argparse import os import json +import sys +from pathlib import Path + +# Hackery to enable importing of utils from ci/scripts/jenkins +REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent +sys.path.append(str(REPO_ROOT / "ci" / "scripts" / "jenkins")) from git_utils import git, GitHubRepo, parse_remote, DRY_RUN from cmd_utils import init_log diff --git a/ci/scripts/github_skipped_tests_comment.py b/ci/scripts/github/github_skipped_tests_comment.py similarity index 99% rename from ci/scripts/github_skipped_tests_comment.py rename to ci/scripts/github/github_skipped_tests_comment.py index 46e579105324..24c27a3cb0e6 100755 --- a/ci/scripts/github_skipped_tests_comment.py +++ b/ci/scripts/github/github_skipped_tests_comment.py @@ -142,7 +142,7 @@ def build_comment( text += "\n" text += ( f"Additional tests that were skipped in the CI build and present in the [`required_tests_to_run`]" - f"(https://github.com/apache/tvm/blob/main/ci/scripts/required_tests_to_run.json) file:" + f"(https://github.com/apache/tvm/blob/main/ci/scripts/github/required_tests_to_run.json) file:" f"\n```\n" ) for skip in additional_skipped_list: diff --git a/ci/scripts/github_tag_teams.py b/ci/scripts/github/github_tag_teams.py similarity index 97% rename from ci/scripts/github_tag_teams.py rename to ci/scripts/github/github_tag_teams.py index fd63070db1ba..e50efa301d88 100755 --- a/ci/scripts/github_tag_teams.py +++ b/ci/scripts/github/github_tag_teams.py @@ -21,8 +21,13 @@ import argparse import logging import re +import sys +from pathlib import Path from typing import Dict, Any, List, Tuple, Optional +# Hackery to enable importing of utils from ci/scripts/jenkins +REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent +sys.path.append(str(REPO_ROOT / "ci" / "scripts" / "jenkins")) from git_utils import git, GitHubRepo, parse_remote, find_ccs, dry_run_token from cmd_utils import tags_from_title, init_log diff --git a/ci/scripts/github_tvmbot.py b/ci/scripts/github/github_tvmbot.py similarity index 99% rename from ci/scripts/github_tvmbot.py rename to ci/scripts/github/github_tvmbot.py index ee9607dd0254..908551bdec0d 100755 --- a/ci/scripts/github_tvmbot.py +++ b/ci/scripts/github/github_tvmbot.py @@ -19,6 +19,7 @@ import os import json import argparse +import sys import warnings import logging import traceback @@ -26,6 +27,10 @@ from typing import Dict, Any, List, Optional, Callable, Union from pathlib import Path +# Hackery to enable importing of utils from ci/scripts/jenkins +REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent +sys.path.append(str(REPO_ROOT / "ci" / "scripts" / "jenkins")) + from git_utils import git, GitHubRepo, parse_remote, post from cmd_utils import init_log diff --git a/ci/scripts/ping_reviewers.py b/ci/scripts/github/ping_reviewers.py similarity index 96% rename from ci/scripts/ping_reviewers.py rename to ci/scripts/github/ping_reviewers.py index af642a52a0eb..0ecdf76dd014 100755 --- a/ci/scripts/ping_reviewers.py +++ b/ci/scripts/github/ping_reviewers.py @@ -16,15 +16,20 @@ # specific language governing permissions and limitations # under the License. -import os import argparse import re import datetime import json +import sys import textwrap -from typing import Dict, Any, List +from pathlib import Path +from typing import List -from git_utils import git, GitHubRepo, parse_remote +# Hackery to enable importing of utils from ci/scripts/jenkins +REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent +sys.path.append(str(REPO_ROOT / "ci" / "scripts" / "jenkins")) + +from git_utils import git, parse_remote GIT_DATE_FORMAT = "%Y-%m-%dT%H:%M:%SZ" diff --git a/ci/scripts/update_branch.py b/ci/scripts/github/update_branch.py similarity index 96% rename from ci/scripts/update_branch.py rename to ci/scripts/github/update_branch.py index 8f2558742217..9f689f6bfa65 100755 --- a/ci/scripts/update_branch.py +++ b/ci/scripts/github/update_branch.py @@ -19,9 +19,14 @@ import os import json import argparse -import tempfile +import sys +from pathlib import Path from typing import Any, Dict +# Hackery to enable importing of utils from ci/scripts/jenkins +REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent +sys.path.append(str(REPO_ROOT / "ci" / "scripts" / "jenkins")) + from git_utils import git, GitHubRepo, parse_remote diff --git a/ci/scripts/check_pr.py b/ci/scripts/jenkins/check_pr.py similarity index 100% rename from ci/scripts/check_pr.py rename to ci/scripts/jenkins/check_pr.py diff --git a/ci/scripts/cmd_utils.py b/ci/scripts/jenkins/cmd_utils.py similarity index 95% rename from ci/scripts/cmd_utils.py rename to ci/scripts/jenkins/cmd_utils.py index f83ec6f24ecd..52eaf9ac0ad2 100644 --- a/ci/scripts/cmd_utils.py +++ b/ci/scripts/jenkins/cmd_utils.py @@ -24,7 +24,8 @@ from typing import List -REPO_ROOT = Path(__file__).resolve().parent.parent.parent +REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent +assert (REPO_ROOT / "Jenkinsfile").exists class RelativePathFilter(logging.Filter): diff --git a/ci/scripts/determine_docker_images.py b/ci/scripts/jenkins/determine_docker_images.py similarity index 98% rename from ci/scripts/determine_docker_images.py rename to ci/scripts/jenkins/determine_docker_images.py index dbcde82cff7a..82acf2ea46b4 100755 --- a/ci/scripts/determine_docker_images.py +++ b/ci/scripts/jenkins/determine_docker_images.py @@ -26,8 +26,7 @@ from http_utils import get -from cmd_utils import init_log, REPO_ROOT - +from cmd_utils import init_log DOCKER_API_BASE = "https://hub.docker.com/v2/" PAGE_SIZE = 25 diff --git a/ci/scripts/git_change_docker.sh b/ci/scripts/jenkins/git_change_docker.sh similarity index 100% rename from ci/scripts/git_change_docker.sh rename to ci/scripts/jenkins/git_change_docker.sh diff --git a/ci/scripts/git_change_docs.sh b/ci/scripts/jenkins/git_change_docs.sh similarity index 100% rename from ci/scripts/git_change_docs.sh rename to ci/scripts/jenkins/git_change_docs.sh diff --git a/ci/scripts/git_skip_ci.py b/ci/scripts/jenkins/git_skip_ci.py similarity index 100% rename from ci/scripts/git_skip_ci.py rename to ci/scripts/jenkins/git_skip_ci.py diff --git a/ci/scripts/git_skip_ci_globs.py b/ci/scripts/jenkins/git_skip_ci_globs.py similarity index 100% rename from ci/scripts/git_skip_ci_globs.py rename to ci/scripts/jenkins/git_skip_ci_globs.py diff --git a/ci/scripts/git_utils.py b/ci/scripts/jenkins/git_utils.py similarity index 100% rename from ci/scripts/git_utils.py rename to ci/scripts/jenkins/git_utils.py diff --git a/ci/scripts/http_utils.py b/ci/scripts/jenkins/http_utils.py similarity index 100% rename from ci/scripts/http_utils.py rename to ci/scripts/jenkins/http_utils.py diff --git a/ci/scripts/open_docker_update_pr.py b/ci/scripts/jenkins/open_docker_update_pr.py similarity index 99% rename from ci/scripts/open_docker_update_pr.py rename to ci/scripts/jenkins/open_docker_update_pr.py index 516c8c1a7d8c..9dcb241d5fd8 100755 --- a/ci/scripts/open_docker_update_pr.py +++ b/ci/scripts/jenkins/open_docker_update_pr.py @@ -17,7 +17,6 @@ # under the License. import argparse -import re import logging import datetime import os @@ -26,7 +25,7 @@ from urllib import error from typing import List, Dict, Any, Optional, Callable from git_utils import git, parse_remote, GitHubRepo -from cmd_utils import REPO_ROOT, init_log, Sh +from cmd_utils import REPO_ROOT, init_log from should_rebuild_docker import docker_api JENKINSFILE = REPO_ROOT / "ci" / "jenkins" / "Jenkinsfile.j2" diff --git a/ci/scripts/pytest_ids.py b/ci/scripts/jenkins/pytest_ids.py similarity index 100% rename from ci/scripts/pytest_ids.py rename to ci/scripts/jenkins/pytest_ids.py diff --git a/ci/scripts/pytest_wrapper.py b/ci/scripts/jenkins/pytest_wrapper.py similarity index 98% rename from ci/scripts/pytest_wrapper.py rename to ci/scripts/jenkins/pytest_wrapper.py index 4c4410bedc9c..4cc988f5bf72 100755 --- a/ci/scripts/pytest_wrapper.py +++ b/ci/scripts/jenkins/pytest_wrapper.py @@ -18,7 +18,6 @@ import argparse import textwrap import junitparser -import traceback from pathlib import Path from typing import List, Optional import os @@ -28,7 +27,7 @@ from cmd_utils import init_log -REPO_ROOT = Path(__file__).resolve().parent.parent.parent +REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent def lstrip(s: str, prefix: str) -> str: diff --git a/ci/scripts/retry.sh b/ci/scripts/jenkins/retry.sh similarity index 100% rename from ci/scripts/retry.sh rename to ci/scripts/jenkins/retry.sh diff --git a/ci/scripts/should_rebuild_docker.py b/ci/scripts/jenkins/should_rebuild_docker.py similarity index 100% rename from ci/scripts/should_rebuild_docker.py rename to ci/scripts/jenkins/should_rebuild_docker.py diff --git a/ci/scripts/should_run_slow_tests.py b/ci/scripts/jenkins/should_run_slow_tests.py similarity index 95% rename from ci/scripts/should_run_slow_tests.py rename to ci/scripts/jenkins/should_run_slow_tests.py index 40ce068520b8..04bf9b0a542f 100755 --- a/ci/scripts/should_run_slow_tests.py +++ b/ci/scripts/jenkins/should_run_slow_tests.py @@ -17,13 +17,9 @@ # under the License. import os -import json import argparse -import subprocess -import re import textwrap -from urllib import request -from typing import Dict, Tuple, Any, List, Optional +from typing import Tuple, List, Optional from git_utils import GitHubRepo, parse_remote, git diff --git a/ci/scripts/required_tests_to_run.json b/ci/scripts/required_tests_to_run.json deleted file mode 100644 index 8bd265c04fe0..000000000000 --- a/ci/scripts/required_tests_to_run.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "unittest_GPU": - [ - "ctypes.tests.python.unittest.test_meta_schedule_integration#test_meta_schedule_integration_extract_from_bert_base", - "cython.tests.python.unittest.test_meta_schedule_integration#test_meta_schedule_integration_extract_from_bert_base", - "ctypes.tests.python.unittest.test_meta_schedule_integration#test_meta_schedule_dynamic_loop_extent", - "cython.tests.python.unittest.test_meta_schedule_integration#test_meta_schedule_dynamic_loop_extent", - "ctypes.tests.python.unittest.test_meta_schedule_integration#test_extract_task_arm_conv2d_nchwc", - "cython.tests.python.unittest.test_meta_schedule_integration#test_extract_task_arm_conv2d_nchwc" - ] -} diff --git a/docker/bash.sh b/docker/bash.sh index 5ee772867976..5973c7013b85 100755 --- a/docker/bash.sh +++ b/docker/bash.sh @@ -305,7 +305,8 @@ if [ -n "${EXPANDED_SHORTCUT}" ]; then if [ "${CI+x}" == "x" ]; then DOCKER_IMAGE_NAME="${EXPANDED_SHORTCUT}" else - python3 ci/scripts/determine_docker_images.py "$DOCKER_IMAGE_NAME=$EXPANDED_SHORTCUT" 2> /dev/null + python3 ci/scripts/jenkins/determine_docker_images.py "$DOCKER_IMAGE_NAME=$EXPANDED_SHORTCUT" 2> /dev/null + echo "HERE HERE HERE" DOCKER_IMAGE_NAME=$(cat ".docker-image-names/$DOCKER_IMAGE_NAME") if [[ "$DOCKER_IMAGE_NAME" == *"tlcpackstaging"* ]]; then echo "WARNING: resolved docker image to fallback tag in tlcpackstaging" >&2 diff --git a/tests/python/ci/test_ci.py b/tests/python/ci/test_ci.py index 7b7b7298d8b3..710f152c9b1e 100644 --- a/tests/python/ci/test_ci.py +++ b/tests/python/ci/test_ci.py @@ -26,13 +26,15 @@ import pytest import tvm.testing -from .test_utils import REPO_ROOT, TempGit, run_script +from .test_utils import REPO_ROOT, GITHUB_SCRIPT_ROOT, JENKINS_SCRIPT_ROOT, TempGit, run_script # pylint: disable=wrong-import-position,wrong-import-order sys.path.insert(0, str(REPO_ROOT / "ci")) -sys.path.insert(0, str(REPO_ROOT / "ci" / "scripts")) +sys.path.insert(0, str(JENKINS_SCRIPT_ROOT)) +sys.path.insert(0, str(GITHUB_SCRIPT_ROOT)) -import scripts +import scripts.github +import scripts.jenkins # pylint: enable=wrong-import-position,wrong-import-order @@ -132,7 +134,7 @@ def parameterize_named(**kwargs): "jenkins_prefix": "ci.tlcpack.ai", "common_main_build": """{"build_number": "4115", "state": "success"}""", "commit_sha": "sha1234", - "expected_body": "The list below shows tests that ran in main sha1234 but were skipped in the CI build of sha1234:\n```\nunittest -> ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner\nunittest -> ctypes.tests.python.unittest.test_roofline#test_estimate_peak_bandwidth[cuda]\n```\n\nAdditional tests that were skipped in the CI build and present in the [`required_tests_to_run`](https://github.com/apache/tvm/blob/main/ci/scripts/required_tests_to_run.json) file:\n```\nunittest -> ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner\n```\nA detailed report of ran tests is [here](https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/testReport/).", + "expected_body": "The list below shows tests that ran in main sha1234 but were skipped in the CI build of sha1234:\n```\nunittest -> ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner\nunittest -> ctypes.tests.python.unittest.test_roofline#test_estimate_peak_bandwidth[cuda]\n```\n\nAdditional tests that were skipped in the CI build and present in the [`required_tests_to_run`](https://github.com/apache/tvm/blob/main/ci/scripts/github/required_tests_to_run.json) file:\n```\nunittest -> ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner\n```\nA detailed report of ran tests is [here](https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/testReport/).", }, "no-diff": { "main_xml_file": "unittest/file1.xml", @@ -212,7 +214,7 @@ def parameterize_named(**kwargs): "jenkins_prefix": "ci.tlcpack.ai", "common_main_build": """{"build_number": "4115", "state": "success"}""", "commit_sha": "sha1234", - "expected_body": "No diff in skipped tests with main found in this branch for commit sha1234.\n\nAdditional tests that were skipped in the CI build and present in the [`required_tests_to_run`](https://github.com/apache/tvm/blob/main/ci/scripts/required_tests_to_run.json) file:\n```\nunittest -> ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner\n```\nA detailed report of ran tests is [here](https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/testReport/).", + "expected_body": "No diff in skipped tests with main found in this branch for commit sha1234.\n\nAdditional tests that were skipped in the CI build and present in the [`required_tests_to_run`](https://github.com/apache/tvm/blob/main/ci/scripts/github/required_tests_to_run.json) file:\n```\nunittest -> ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner\n```\nA detailed report of ran tests is [here](https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/testReport/).", }, "unable-to-run": { "main_xml_file": "unittest/file1.xml", @@ -300,7 +302,7 @@ def write_xml_file(root_dir, xml_file, xml_content): } } with caplog.at_level(logging.INFO): - comment = scripts.github_skipped_tests_comment.get_skipped_tests_comment( + comment = scripts.github.github_skipped_tests_comment.get_skipped_tests_comment( pr=pr_data, github=None, s3_prefix=s3_prefix, @@ -350,7 +352,7 @@ def test_docs_comment(target_url, base_url, commit_sha, expected_body): ] } } - comment = scripts.github_docs_comment.get_doc_url( + comment = scripts.github.github_docs_comment.get_doc_url( pr=pr_data, base_docs_url=base_url, ) @@ -414,7 +416,7 @@ def test_cc_reviewers( """ Test that reviewers are added from 'cc @someone' messages in PRs """ - reviewers_script = REPO_ROOT / "ci" / "scripts" / "github_cc_reviewers.py" + reviewers_script = GITHUB_SCRIPT_ROOT / "github_cc_reviewers.py" git = TempGit(tmpdir_factory.mktemp("tmp_git_dir")) reviews = [{"user": {"login": r}} for r in existing_review_users] @@ -497,7 +499,7 @@ def test_update_branch(tmpdir_factory, statuses, expected_rc, expected_output): """ Test that the last-successful branch script updates successfully """ - update_script = REPO_ROOT / "ci" / "scripts" / "update_branch.py" + update_script = GITHUB_SCRIPT_ROOT / "update_branch.py" git = TempGit(tmpdir_factory.mktemp("tmp_git_dir")) commit = { @@ -608,7 +610,7 @@ def test_pr_comment(tmpdir_factory, pr_author, comments, expected): """ Test the PR commenting bot """ - comment_script = REPO_ROOT / "ci" / "scripts" / "github_pr_comment.py" + comment_script = GITHUB_SCRIPT_ROOT / "github_pr_comment.py" git = TempGit(tmpdir_factory.mktemp("tmp_git_dir")) target_url = "https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/display/redirect" @@ -739,7 +741,7 @@ def test_skip_ci(tmpdir_factory, commands, should_skip, pr_title, why): """ Test that CI is skipped when it should be """ - skip_ci_script = REPO_ROOT / "ci" / "scripts" / "git_skip_ci.py" + skip_ci_script = JENKINS_SCRIPT_ROOT / "git_skip_ci.py" git = TempGit(tmpdir_factory.mktemp("tmp_git_dir")) @@ -776,7 +778,7 @@ def test_skip_globs(tmpdir_factory, files, should_skip): """ Test that CI is skipped if only certain files are edited """ - script = REPO_ROOT / "ci" / "scripts" / "git_skip_ci_globs.py" + script = JENKINS_SCRIPT_ROOT / "git_skip_ci_globs.py" git = TempGit(tmpdir_factory.mktemp("tmp_git_dir")) @@ -877,7 +879,7 @@ def test_ping_reviewers(tmpdir_factory, pull_request, check): """ Test that reviewers are messaged after a time period of inactivity """ - reviewers_script = REPO_ROOT / "ci" / "scripts" / "ping_reviewers.py" + reviewers_script = GITHUB_SCRIPT_ROOT / "ping_reviewers.py" git = TempGit(tmpdir_factory.mktemp("tmp_git_dir")) @@ -1116,7 +1118,7 @@ def test_github_tag_teams(tmpdir_factory, source_type, data, check): """ Check that individuals are tagged from team headers """ - tag_script = REPO_ROOT / "ci" / "scripts" / "github_tag_teams.py" + tag_script = GITHUB_SCRIPT_ROOT / "github_tag_teams.py" git = TempGit(tmpdir_factory.mktemp("tmp_git_dir")) @@ -1241,7 +1243,7 @@ def test_open_docker_update_pr( tmpdir_factory, tlcpackstaging_body, tlcpack_body, expected, expected_images ): """Test workflow to open a PR to update Docker images""" - tag_script = REPO_ROOT / "ci" / "scripts" / "open_docker_update_pr.py" + tag_script = JENKINS_SCRIPT_ROOT / "open_docker_update_pr.py" git = TempGit(tmpdir_factory.mktemp("tmp_git_dir")) git.run("config", "user.name", "ci") @@ -1300,7 +1302,7 @@ def test_open_docker_update_pr( ) def test_determine_docker_images(tmpdir_factory, images, expected): """Test script to decide whether to use tlcpack or tlcpackstaging for images""" - script = REPO_ROOT / "ci" / "scripts" / "determine_docker_images.py" + script = JENKINS_SCRIPT_ROOT / "determine_docker_images.py" git_dir = tmpdir_factory.mktemp("tmp_git_dir") @@ -1355,7 +1357,7 @@ def test_should_rebuild_docker(tmpdir_factory, changed_files, name, check, expec """ Check that the Docker images are built when necessary """ - tag_script = REPO_ROOT / "ci" / "scripts" / "should_rebuild_docker.py" + tag_script = JENKINS_SCRIPT_ROOT / "should_rebuild_docker.py" git = TempGit(tmpdir_factory.mktemp("tmp_git_dir")) git.run("config", "user.name", "ci") @@ -1434,7 +1436,7 @@ def test_pr_linter(title, body, expected, expected_code): """ Test the PR linter """ - tag_script = REPO_ROOT / "ci" / "scripts" / "check_pr.py" + tag_script = JENKINS_SCRIPT_ROOT / "check_pr.py" pr_data = { "title": title, "body": body, diff --git a/tests/python/ci/test_tvmbot.py b/tests/python/ci/test_tvmbot.py index ceabd46a9b03..de3ab9bb501b 100644 --- a/tests/python/ci/test_tvmbot.py +++ b/tests/python/ci/test_tvmbot.py @@ -23,7 +23,7 @@ from typing import Dict, Any import tvm -from .test_utils import REPO_ROOT, TempGit, run_script +from .test_utils import GITHUB_SCRIPT_ROOT, TempGit, run_script SUCCESS_EXPECTED_OUTPUT = """ @@ -51,7 +51,7 @@ def test(self, tmpdir_factory): """ Run the tvm-bot script using the data from preprocess_data """ - mergebot_script = REPO_ROOT / "ci" / "scripts" / "github_tvmbot.py" + mergebot_script = GITHUB_SCRIPT_ROOT / "github_tvmbot.py" test_json_dir = Path(__file__).resolve().parent / "sample_prs" with open(test_json_dir / f"pr{self.NUMBER}.json") as f: test_data = json.load(f) diff --git a/tests/python/ci/test_utils.py b/tests/python/ci/test_utils.py index 4a0f2710e74a..107d16c11bc1 100644 --- a/tests/python/ci/test_utils.py +++ b/tests/python/ci/test_utils.py @@ -22,6 +22,8 @@ from typing import List, Any REPO_ROOT = pathlib.Path(__file__).resolve().parent.parent.parent.parent +GITHUB_SCRIPT_ROOT = REPO_ROOT / "ci" / "scripts" / "github" +JENKINS_SCRIPT_ROOT = REPO_ROOT / "ci" / "scripts" / "jenkins" class TempGit: diff --git a/tests/scripts/release/gather_prs.py b/tests/scripts/release/gather_prs.py index 5fbfa2278feb..8f98076eb997 100644 --- a/tests/scripts/release/gather_prs.py +++ b/tests/scripts/release/gather_prs.py @@ -25,7 +25,8 @@ from typing import Callable, Dict, List, Any REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent -sys.path.append(str(REPO_ROOT / "ci" / "scripts")) +sys.path.append(str(REPO_ROOT / "ci" / "scripts" / "jenkins")) +sys.path.append(str(REPO_ROOT / "ci" / "scripts" / "github")) from git_utils import git, GitHubRepo from github_tag_teams import tags_from_title diff --git a/tests/scripts/release/make_notes.py b/tests/scripts/release/make_notes.py index 95cb15197275..f39c1c567ceb 100644 --- a/tests/scripts/release/make_notes.py +++ b/tests/scripts/release/make_notes.py @@ -27,6 +27,8 @@ REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent sys.path.append(str(REPO_ROOT / "tests" / "scripts")) +sys.path.append(str(REPO_ROOT / "tests" / "scripts" / "github")) +sys.path.append(str(REPO_ROOT / "tests" / "scripts" / "jenkins")) def strip_header(title: str, header: str) -> str: diff --git a/tests/scripts/setup-pytest-env.sh b/tests/scripts/setup-pytest-env.sh index 305f626d666c..fbb1ad4cbd3e 100755 --- a/tests/scripts/setup-pytest-env.sh +++ b/tests/scripts/setup-pytest-env.sh @@ -39,7 +39,7 @@ function cleanup() { set +x if [ "${#pytest_errors[@]}" -gt 0 ]; then echo "These pytest invocations failed, the results can be found in the Jenkins 'Tests' tab or by scrolling up through the raw logs here." - python3 ci/scripts/pytest_wrapper.py "${pytest_errors[@]}" + python3 ci/scripts/jenkins/pytest_wrapper.py "${pytest_errors[@]}" exit 1 fi set -x diff --git a/tests/scripts/task_build.py b/tests/scripts/task_build.py index 1a8a1d112fc0..157e8195ce78 100755 --- a/tests/scripts/task_build.py +++ b/tests/scripts/task_build.py @@ -24,9 +24,9 @@ from pathlib import Path -# Hackery to enable importing of utils from ci/scripts +# Hackery to enable importing of utils from ci/scripts/jenkins REPO_ROOT = Path(__file__).resolve().parent.parent.parent -sys.path.append(str(REPO_ROOT / "ci" / "scripts")) +sys.path.append(str(REPO_ROOT / "ci" / "scripts" / "jenkins")) from cmd_utils import Sh, init_log, REPO_ROOT diff --git a/tests/scripts/task_python_frontend.sh b/tests/scripts/task_python_frontend.sh index 61d7238a594b..ee6be87b36d0 100755 --- a/tests/scripts/task_python_frontend.sh +++ b/tests/scripts/task_python_frontend.sh @@ -42,7 +42,7 @@ run_pytest cython python-frontend-pytorch tests/python/frontend/pytorch echo "Running relay Tensorflow frontend test..." # Note: Tensorflow tests often have memory issues, so invoke each one separately -TENSORFLOW_TESTS=$(./ci/scripts/pytest_ids.py --folder tests/python/frontend/tensorflow) +TENSORFLOW_TESTS=$(./ci/scripts/jenkins/pytest_ids.py --folder tests/python/frontend/tensorflow) i=0 for node_id in $TENSORFLOW_TESTS; do echo "$node_id"