[CI] Separate the ci scripts into Github and Jenkins scripts (#13368)

This PR is a duplicate of #12940 and #12941. For some reason, I am unable to reopen #12940.

.github/disabled_workflows/pr_comment_bot.yml

@@ -38,7 +38,7 @@ jobs:
           PR_NUMBER: ${{ github.event.number }}
         run: |
           set -eux
-          python ci/scripts/github_pr_comment.py --pr "$PR_NUMBER"
+          python ci/scripts/github/github_pr_comment.py --pr "$PR_NUMBER"
       - name: Comment bot comment (status)
         if: ${{ github.event.state }}
         env:
@@ -49,7 +49,7 @@ jobs:
           if [[ "$URL" == *"PR-"* ]]; then
             echo "PR status, sending comment"
             PR_NUMBER=$(echo $URL | sed 's/.*PR-//g' | sed 's/\/.*//g')
-            python ci/scripts/github_pr_comment.py --pr "$PR_NUMBER" || /bin/true
+            python ci/scripts/github/github_pr_comment.py --pr "$PR_NUMBER" || /bin/true
           else
             echo "Not a PR status, skipping"
           fi

.github/workflows/cc_bot.yml

@@ -44,4 +44,4 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           set -eux
-          python ci/scripts/github_cc_reviewers.py || echo step failed
+          python ci/scripts/github/github_cc_reviewers.py || echo step failed

.github/workflows/nightly_docker_update.yml

@@ -28,4 +28,4 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           set -eux
-          python ci/scripts/open_docker_update_pr.py
+          python ci/scripts/jenkins/open_docker_update_pr.py

.github/workflows/ping_reviewers.yml

@@ -20,4 +20,4 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           set -eux
-          python ci/scripts/ping_reviewers.py --wait-time-minutes 10080 || echo failed
+          python ci/scripts/github/ping_reviewers.py --wait-time-minutes 10080 || echo failed

.github/workflows/tag_teams.yml

@@ -38,4 +38,4 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           set -eux
-          python ci/scripts/github_tag_teams.py || echo failed
+          python ci/scripts/github/github_tag_teams.py || echo failed

.github/workflows/tvmbot.yml

@@ -34,4 +34,4 @@ jobs:
           RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
         run: |
           set -eux
-          python ci/scripts/github_tvmbot.py --pr "$PR_NUMBER" --run-url "$RUN_URL" --trigger-comment-json "$ISSUE_COMMENT"
+          python ci/scripts/github/github_tvmbot.py --pr "$PR_NUMBER" --run-url "$RUN_URL" --trigger-comment-json "$ISSUE_COMMENT"

.github/workflows/update_last_successful_branch.yml

@@ -41,4 +41,4 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           set -eux
-          python ci/scripts/update_branch.py || echo step failed
+          python ci/scripts/github/update_branch.py || echo step failed

ci/jenkins/Deploy.groovy.j2

@@ -30,7 +30,7 @@ def update_docker(ecr_image, hub_image) {
   sh(
     script: """
     set -eux
-    . ci/scripts/retry.sh
+    . ${jenkins_scripts_root}/retry.sh
     docker tag \
       ${ecr_image} \
       ${hub_image}
@@ -148,7 +148,7 @@ def deploy() {
                 sh(
                   script: """
                     set -eux
-                    . ci/scripts/retry.sh
+                    . ${jenkins_scripts_root}/retry.sh
                     docker pull tlcpackstaging/{{ image.name }}:${tag}
                     docker tag tlcpackstaging/{{ image.name }}:${tag} tlcpack/{{ image.name.replace("_", "-") }}:${tag}
                     retry 5 docker push tlcpack/{{ image.name.replace("_", "-") }}:${tag}

ci/jenkins/DockerBuild.groovy.j2

@@ -21,7 +21,7 @@ def ecr_push(full_name) {
       sh(
         script: """
           set -x
-          . ci/scripts/retry.sh
+          . ${jenkins_scripts_root}/retry.sh
           docker tag ${full_name} \$AWS_ECR_REPO/${full_name}
           retry 5 docker push \$AWS_ECR_REPO/${full_name}
         """,
@@ -64,7 +64,7 @@ def ecr_pull(full_name) {
       sh(
         script: """
           set -eux
-          . ci/scripts/retry.sh
+          . ${jenkins_scripts_root}/retry.sh
           retry 5 docker pull ${full_name}
         """,
         label: 'Pull image from ECR'

ci/jenkins/Jenkinsfile.j2

@@ -100,6 +100,8 @@ rebuild_docker_images = false
 {% set hexagon_api = ['build/hexagon_api_output',] %}
 s3_prefix = "tvm-jenkins-artifacts-prod/tvm/${env.BRANCH_NAME}/${env.BUILD_NUMBER}"
 
+// Jenkins script root directory
+jenkins_scripts_root = "ci/scripts/jenkins"
 {% set aws_default_region = "us-west-2" %}
 {% set aws_ecr_url = "dkr.ecr." + aws_default_region + ".amazonaws.com" %}
 

ci/jenkins/Prepare.groovy.j2

@@ -34,7 +34,7 @@ def init_git() {
   sh(
     script: """
       set -eux
-      . ci/scripts/retry.sh
+      . ${jenkins_scripts_root}/retry.sh
       retry 3 timeout 5m git submodule update --init -f --jobs 0
     """,
     label: 'Update git submodules',
@@ -66,7 +66,7 @@ def docker_init(image) {
     sh(
       script: """
       set -eux
-      . ci/scripts/retry.sh
+      . ${jenkins_scripts_root}/retry.sh
       retry 5 docker pull ${image}
       """,
       label: 'Pull docker image',
@@ -82,7 +82,7 @@ def should_skip_slow_tests(pr_number) {
     // Exit code of 1 means run slow tests, exit code of 0 means skip slow tests
     result = sh (
       returnStatus: true,
-      script: "./ci/scripts/should_run_slow_tests.py --pr '${pr_number}'",
+      script: "./${jenkins_scripts_root}/should_run_slow_tests.py --pr '${pr_number}'",
       label: 'Check if CI should run slow tests',
     )
   }
@@ -118,7 +118,7 @@ def checkout_trusted_files() {
     // (especially those that access secrets) should be checked out here so
     // only trusted versions are used in CI
     sh(
-      script: "git checkout ${upstream_revision} ci/scripts/.",
+      script: "git checkout ${upstream_revision} ${jenkins_scripts_root}/.",
       label: 'Check out trusted files',
     )
   }
@@ -131,7 +131,7 @@ def should_skip_ci(pr_number) {
   }
   glob_skip_ci_code = sh (
     returnStatus: true,
-    script: "./ci/scripts/git_skip_ci_globs.py",
+    script: "./${jenkins_scripts_root}/git_skip_ci_globs.py",
     label: 'Check if CI should be skipped due to changed files',
   )
   if (glob_skip_ci_code == 0) {
@@ -145,7 +145,7 @@ def should_skip_ci(pr_number) {
     // full CI just in case). Exit code of 0 means skip CI.
     git_skip_ci_code = sh (
       returnStatus: true,
-      script: "./ci/scripts/git_skip_ci.py --pr '${pr_number}'",
+      script: "./${jenkins_scripts_root}/git_skip_ci.py --pr '${pr_number}'",
       label: 'Check if CI should be skipped',
     )
   }
@@ -162,7 +162,7 @@ def check_pr(pr_number) {
     variable: 'GITHUB_TOKEN',
     )]) {
     sh (
-      script: "python3 ci/scripts/check_pr.py --pr ${pr_number}",
+      script: "python3 ${jenkins_scripts_root}/check_pr.py --pr ${pr_number}",
       label: 'Check PR title and body',
     )
   }
@@ -179,7 +179,7 @@ def prepare() {
 
         if (env.DETERMINE_DOCKER_IMAGES == 'yes') {
           sh(
-            script: "./ci/scripts/determine_docker_images.py {% for image in images %}{{ image.name }}={% raw %}${{% endraw %}{{ image.name }}{% raw %}}{% endraw %} {% endfor %}",
+            script: "./${jenkins_scripts_root}/determine_docker_images.py {% for image in images %}{{ image.name }}={% raw %}${{% endraw %}{{ image.name }}{% raw %}}{% endraw %} {% endfor %}",
             label: 'Decide whether to use tlcpack or tlcpackstaging for Docker images',
           )
           // Pull image names from the results of should_rebuild_docker.py
@@ -205,14 +205,14 @@ def prepare() {
 
         is_docs_only_build = sh (
           returnStatus: true,
-          script: './ci/scripts/git_change_docs.sh',
+          script: './${jenkins_scripts_root}/git_change_docs.sh',
           label: 'Check for docs only changes',
         )
         skip_ci = should_skip_ci(env.CHANGE_ID)
         skip_slow_tests = should_skip_slow_tests(env.CHANGE_ID)
         rebuild_docker_images = sh (
           returnStatus: true,
-          script: './ci/scripts/git_change_docker.sh',
+          script: './${jenkins_scripts_root}/git_change_docker.sh',
           label: 'Check for any docker changes',
         )
 

ci/jenkins/macros.j2

@@ -191,7 +191,7 @@ def {{ method_name }}() {
 sh(
       script: """
         set -eux
-        . ci/scripts/retry.sh
+        . ${jenkins_scripts_root}/retry.sh
         {% for filename in filenames %}
         md5sum {{ filename }}
         retry 3 aws s3 cp --no-progress {{ filename }} s3://${s3_prefix}/{{ tag }}/{{ filename }}
@@ -208,7 +208,7 @@ sh(
 sh(
             script: """
               set -eux
-              . ci/scripts/retry.sh
+              . ${jenkins_scripts_root}/retry.sh
               {% for filename in filenames %}
               retry 3 aws s3 cp --no-progress s3://${s3_prefix}/{{ tag }}/{{ filename }} {{ filename }}
               md5sum {{ filename }}

ci/scripts/__init__.py → ci/scripts/github/__init__.py

@@ -14,6 +14,6 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-"""Package to enable testing of CI scripts"""
+"""Package to enable testing of Github scripts"""
 
 from . import github_skipped_tests_comment, github_pr_comment, github_tag_teams, github_docs_comment

ci/scripts/github_cc_reviewers.py → ci/scripts/github/github_cc_reviewers.py

@@ -21,9 +21,13 @@
 import json
 import argparse
 import re
+from pathlib import Path
 from urllib import error
 from typing import Dict, Any, List
 
+# Hackery to enable importing of utils from ci/scripts/jenkins
+REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent
+sys.path.append(str(REPO_ROOT / "ci" / "scripts" / "jenkins"))
 
 from git_utils import git, GitHubRepo, parse_remote
 

ci/scripts/github_commenter.py → ci/scripts/github/github_commenter.py

@@ -18,8 +18,14 @@
 
 import re
 import logging
+import sys
+from pathlib import Path
 from typing import Dict, Tuple, Any, Optional, List, Union
 
+# Hackery to enable importing of utils from ci/scripts/jenkins
+REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent
+sys.path.append(str(REPO_ROOT / "ci" / "scripts" / "jenkins"))
+
 from git_utils import GitHubRepo
 
 BOT_COMMENT_START = "<!---bot-comment-->"

ci/scripts/github_pr_comment.py → ci/scripts/github/github_pr_comment.py

@@ -18,6 +18,12 @@
 import argparse
 import os
 import json
+import sys
+from pathlib import Path
+
+# Hackery to enable importing of utils from ci/scripts/jenkins
+REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent
+sys.path.append(str(REPO_ROOT / "ci" / "scripts" / "jenkins"))
 
 from git_utils import git, GitHubRepo, parse_remote, DRY_RUN
 from cmd_utils import init_log

ci/scripts/github_skipped_tests_comment.py → ci/scripts/github/github_skipped_tests_comment.py

@@ -142,7 +142,7 @@ def build_comment(
         text += "\n"
         text += (
             f"Additional tests that were skipped in the CI build and present in the [`required_tests_to_run`]"
-            f"(https://github.com/apache/tvm/blob/main/ci/scripts/required_tests_to_run.json) file:"
+            f"(https://github.com/apache/tvm/blob/main/ci/scripts/github/required_tests_to_run.json) file:"
             f"\n```\n"
         )
         for skip in additional_skipped_list:

ci/scripts/github_tag_teams.py → ci/scripts/github/github_tag_teams.py

@@ -21,8 +21,13 @@
 import argparse
 import logging
 import re
+import sys
+from pathlib import Path
 from typing import Dict, Any, List, Tuple, Optional
 
+# Hackery to enable importing of utils from ci/scripts/jenkins
+REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent
+sys.path.append(str(REPO_ROOT / "ci" / "scripts" / "jenkins"))
 
 from git_utils import git, GitHubRepo, parse_remote, find_ccs, dry_run_token
 from cmd_utils import tags_from_title, init_log

ci/scripts/github_tvmbot.py → ci/scripts/github/github_tvmbot.py

@@ -19,13 +19,18 @@
 import os
 import json
 import argparse
+import sys
 import warnings
 import logging
 import traceback
 import re
 from typing import Dict, Any, List, Optional, Callable, Union
 from pathlib import Path
 
+# Hackery to enable importing of utils from ci/scripts/jenkins
+REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent
+sys.path.append(str(REPO_ROOT / "ci" / "scripts" / "jenkins"))
+
 from git_utils import git, GitHubRepo, parse_remote, post
 from cmd_utils import init_log
 

ci/scripts/ping_reviewers.py → ci/scripts/github/ping_reviewers.py

@@ -16,15 +16,20 @@
 # specific language governing permissions and limitations
 # under the License.
 
-import os
 import argparse
 import re
 import datetime
 import json
+import sys
 import textwrap
-from typing import Dict, Any, List
+from pathlib import Path
+from typing import List
 
-from git_utils import git, GitHubRepo, parse_remote
+# Hackery to enable importing of utils from ci/scripts/jenkins
+REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent
+sys.path.append(str(REPO_ROOT / "ci" / "scripts" / "jenkins"))
+
+from git_utils import git, parse_remote
 
 GIT_DATE_FORMAT = "%Y-%m-%dT%H:%M:%SZ"
 

ci/scripts/update_branch.py → ci/scripts/github/update_branch.py

@@ -19,9 +19,14 @@
 import os
 import json
 import argparse
-import tempfile
+import sys
+from pathlib import Path
 from typing import Any, Dict
 
+# Hackery to enable importing of utils from ci/scripts/jenkins
+REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent
+sys.path.append(str(REPO_ROOT / "ci" / "scripts" / "jenkins"))
+
 from git_utils import git, GitHubRepo, parse_remote
 
 

ci/scripts/cmd_utils.py → ci/scripts/jenkins/cmd_utils.py

@@ -24,7 +24,8 @@
 from typing import List
 
 
-REPO_ROOT = Path(__file__).resolve().parent.parent.parent
+REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent
+assert (REPO_ROOT / "Jenkinsfile").exists
 
 
 class RelativePathFilter(logging.Filter):

ci/scripts/determine_docker_images.py → ci/scripts/jenkins/determine_docker_images.py

@@ -26,8 +26,7 @@
 
 
 from http_utils import get
-from cmd_utils import init_log, REPO_ROOT
-
+from cmd_utils import init_log
 
 DOCKER_API_BASE = "https://hub.docker.com/v2/"
 PAGE_SIZE = 25

ci/scripts/open_docker_update_pr.py → ci/scripts/jenkins/open_docker_update_pr.py

@@ -17,7 +17,6 @@
 # under the License.
 
 import argparse
-import re
 import logging
 import datetime
 import os
@@ -26,7 +25,7 @@
 from urllib import error
 from typing import List, Dict, Any, Optional, Callable
 from git_utils import git, parse_remote, GitHubRepo
-from cmd_utils import REPO_ROOT, init_log, Sh
+from cmd_utils import REPO_ROOT, init_log
 from should_rebuild_docker import docker_api
 
 JENKINSFILE = REPO_ROOT / "ci" / "jenkins" / "Jenkinsfile.j2"

ci/scripts/pytest_wrapper.py → ci/scripts/jenkins/pytest_wrapper.py

@@ -18,7 +18,6 @@
 import argparse
 import textwrap
 import junitparser
-import traceback
 from pathlib import Path
 from typing import List, Optional
 import os
@@ -28,7 +27,7 @@
 from cmd_utils import init_log
 
 
-REPO_ROOT = Path(__file__).resolve().parent.parent.parent
+REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent
 
 
 def lstrip(s: str, prefix: str) -> str: