diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 48b9d62bb9b7..313c440cbd21 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -38,59 +38,124 @@ jobs: MacOS: runs-on: macOS-latest steps: - - uses: actions/checkout@v2 - with: - submodules: 'recursive' - - name: Set up environment - uses: ./.github/actions/setup - - name: Conda Build - shell: bash -l {0} - run: >- - conda build --output-folder=conda/pkg conda/recipe && - conda install tvm -c ./conda/pkg - - name: Build iOS RPC - run: | - IOS_VERSION="14.0" - CMAKE_FLAGS="-DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_SYSTEM_NAME=iOS \ - -DCMAKE_SYSTEM_VERSION=${IOS_VERSION} \ - -DCMAKE_OSX_SYSROOT=iphonesimulator \ - -DCMAKE_OSX_ARCHITECTURES=x86_64 \ - -DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \ - -DCMAKE_BUILD_WITH_INSTALL_NAME_DIR=ON \ - -DUSE_IOS_RPC=ON" - - mkdir build-ios-simulator - cd build-ios-simulator - cmake .. ${CMAKE_FLAGS} - cmake --build . --target ios_rpc - - name: Test - shell: bash -l {0} - run: >- - python -m pytest -v tests/python/all-platform-minimal-test - - name: Test iOS RPC - shell: bash -l {0} - run: >- - python -m pip install tornado psutil cloudpickle && - export PYTHONPATH=tests/python/contrib:${PYTHONPATH} && - export BUNDLE_ID=org.apache.tvmrpc && - export BUNDLE_PATH=build-ios-simulator/apps/ios_rpc/ios_rpc/src/ios_rpc-build/Release-iphonesimulator/tvmrpc.app && - python -m pytest -v tests/python/contrib/test_rpc_server_device.py + - uses: actions/checkout@v2 + with: + submodules: 'recursive' + - name: Set up environment + uses: ./.github/actions/setup + - name: Conda Build + shell: bash -l {0} + run: >- + conda build --output-folder=conda/pkg conda/recipe && + conda install tvm -c ./conda/pkg + - name: Build iOS RPC + run: | + IOS_VERSION="14.0" + CMAKE_FLAGS="-DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_SYSTEM_NAME=iOS \ + -DCMAKE_SYSTEM_VERSION=${IOS_VERSION} \ + -DCMAKE_OSX_SYSROOT=iphonesimulator \ + -DCMAKE_OSX_ARCHITECTURES=x86_64 \ + -DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \ + -DCMAKE_BUILD_WITH_INSTALL_NAME_DIR=ON \ + -DUSE_IOS_RPC=ON" + + mkdir build-ios-simulator + cd build-ios-simulator + cmake .. ${CMAKE_FLAGS} + cmake --build . --target ios_rpc + - name: Test + shell: bash -l {0} + run: >- + python -m pytest -v tests/python/all-platform-minimal-test + - name: Test iOS RPC + shell: bash -l {0} + run: >- + python -m pip install tornado psutil cloudpickle && + export PYTHONPATH=tests/python/contrib:${PYTHONPATH} && + export BUNDLE_ID=org.apache.tvmrpc && + export BUNDLE_PATH=build-ios-simulator/apps/ios_rpc/ios_rpc/src/ios_rpc-build/Release-iphonesimulator/tvmrpc.app && + python -m pytest -v tests/python/contrib/test_rpc_server_device.py Windows: runs-on: windows-2019 steps: - - uses: actions/checkout@v2 - with: - submodules: 'recursive' - - name: Set up environment - uses: ./.github/actions/setup - - name: Conda Build - shell: cmd /C call {0} - run: >- - conda build --output-folder=conda/pkg conda/recipe && - conda install tvm -c ./conda/pkg - - name: Test - shell: cmd /C call {0} - run: >- - python -m pytest -v tests/python/all-platform-minimal-test + - uses: actions/checkout@v2 + with: + submodules: 'recursive' + - name: Set up environment + uses: ./.github/actions/setup + - name: Conda Build + shell: cmd /C call {0} + run: >- + conda build --output-folder=conda/pkg conda/recipe && + conda install tvm -c ./conda/pkg + - name: Test + shell: cmd /C call {0} + run: >- + python -m pytest -v tests/python/all-platform-minimal-test + + Android: + runs-on: Ubuntu-20.04 + steps: + - uses: actions/checkout@v2 + with: + submodules: 'recursive' + - name: Set up environment + uses: ./.github/actions/setup + - name: Set up java + uses: actions/setup-java@v3 + with: + distribution: 'zulu' + java-version: '11' + - name: Build TVM + shell: bash -l {0} + run: | + mkdir build + cd build + ../tests/scripts/task_config_build_jvm.sh . + cmake .. + make + - name: Build TVM4J + run: | + make jvmpkg + - name: Build android_rpc + working-directory: apps/android_rpc + run: | + export PATH="${ANDROID_NDK_HOME}:$PATH" + gradle clean build + - name: Upload android_rpc APK + uses: actions/upload-artifact@v2 + with: + name: android_rpc-debug.apk + path: ./apps/android_rpc/app/build/outputs/apk/debug/app-debug.apk + - name: Build android_deploy + working-directory: apps/android_deploy + run: | + export PATH="${ANDROID_NDK_HOME}:$PATH" + gradle clean build + - name: Upload android_deploy APK + uses: actions/upload-artifact@v2 + with: + name: android_deploy-debug.apk + path: ./apps/android_deploy/app/build/outputs/apk/debug/app-debug.apk + - name: Build android_camera + working-directory: apps/android_camera + run: | + mkdir -p app/src/main/assets/models/ + export TVM_NDK_CC=${ANDROID_NDK_HOME}/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android30-clang++ + export TVM_HOME=~/work/tvm/tvm + export PYTHONPATH=$TVM_HOME/python:${PYTHONPATH} + python3 ${TVM_HOME}/python/gen_requirements.py + pip3 install -r ${TVM_HOME}/python/requirements/core.txt + cd models + pip3 install -r requirements.txt + python3 prepare_model.py + cd .. + export PATH="${ANDROID_NDK_HOME}:$PATH" + gradle clean build + - name: Upload android_camera APK + uses: actions/upload-artifact@v2 + with: + name: android_camera-debug.apk + path: ./apps/android_camera/app/build/outputs/apk/debug/app-debug.apk \ No newline at end of file diff --git a/Jenkinsfile b/Jenkinsfile index 0b64f9306844..1e207f09059b 100755 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -45,7 +45,7 @@ // 'python3 jenkins/generate.py' // Note: This timestamp is here to ensure that updates to the Jenkinsfile are // always rebased on main before merging: -// Generated at 2022-05-20T13:24:01.371704 +// Generated at 2022-05-23T16:38:45.963400 import org.jenkinsci.plugins.pipeline.modeldefinition.Utils // NOTE: these lines are scanned by docker/dev_common.sh. Please update the regex as needed. --> @@ -484,6 +484,13 @@ def add_microtvm_permissions() { ) } +def add_hexagon_permissions() { + sh( + script: 'find build/hexagon_api_output -type f | xargs chmod +x', + label: 'Add execute permissions for hexagon files', + ) +} + def build() { stage('Build') { environment { @@ -703,6 +710,10 @@ stage('Build') { label: 'Create Hexagon cmake config', ) make(ci_hexagon, 'build', '-j2') + sh ( + script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_build_hexagon_api.sh", + label: 'Build Hexagon API', + ) sh( script: """ set -eux @@ -712,6 +723,7 @@ stage('Build') { aws s3 cp --no-progress build/libtvm_runtime.so s3://${s3_prefix}/hexagon/build/libtvm_runtime.so md5sum build/config.cmake aws s3 cp --no-progress build/config.cmake s3://${s3_prefix}/hexagon/build/config.cmake + aws s3 cp --no-progress build/hexagon_api_output s3://${s3_prefix}/hexagon/build/hexagon_api_output --recursive """, label: 'Upload artifacts to S3', ) @@ -725,24 +737,24 @@ stage('Build') { ) } } -def test() { -stage('Test') { - environment { - SKIP_SLOW_TESTS = "${skip_slow_tests}" - } - parallel( - 'unittest: GPU 1 of 2': { - if (!skip_ci && is_docs_only_build != 1) { - node('GPU') { - ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-gpu") { - try { - init_git() - timeout(time: max_time, unit: 'MINUTES') { - withEnv([ - 'PLATFORM=gpu', - 'TVM_NUM_SHARDS=2', - 'TVM_SHARD_INDEX=0'], { - sh( + +// We have to do this whacky split of the code from where it's used since the +// JVM limits method length to 64k and we easily exceed that with all this +// autogenerated code. This makes it so each test step is in its own method so +// that each individual method isn't too big. + +def shard_run_unittest_GPU_1_of_3() { + if (!skip_ci && is_docs_only_build != 1) { + node('GPU') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-gpu") { + try { + init_git() + timeout(time: max_time, unit: 'MINUTES') { + withEnv([ + 'PLATFORM=gpu', + 'TVM_NUM_SHARDS=3', + 'TVM_SHARD_INDEX=0'], { + sh( script: """ set -eux aws s3 cp --no-progress s3://${s3_prefix}/gpu2/build/libtvm.so build/libtvm.so @@ -757,9 +769,9 @@ stage('Test') { label: 'Download artifacts from S3', ) - cpp_unittest(ci_gpu) + cpp_unittest(ci_gpu) - sh( + sh( script: """ set -eux aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so @@ -774,39 +786,40 @@ stage('Test') { label: 'Download artifacts from S3', ) - ci_setup(ci_gpu) - cpp_unittest(ci_gpu) - sh ( - script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_unittest_gpuonly.sh", - label: 'Run Python GPU unit tests', - ) - sh ( - script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_integration_gpuonly.sh", - label: 'Run Python GPU integration tests', - ) - }) - } - } finally { - junit 'build/pytest-results/*.xml' + ci_setup(ci_gpu) + cpp_unittest(ci_gpu) + sh ( + script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_unittest_gpuonly.sh", + label: 'Run Python GPU unit tests', + ) + sh ( + script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_integration_gpuonly.sh", + label: 'Run Python GPU integration tests', + ) + }) } + } finally { + junit 'build/pytest-results/*.xml' } } - } else { - Utils.markStageSkippedForConditional('unittest: GPU 1 of 2') } - }, - 'unittest: GPU 2 of 2': { - if (!skip_ci && is_docs_only_build != 1) { - node('GPU') { - ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-gpu") { - try { - init_git() - timeout(time: max_time, unit: 'MINUTES') { - withEnv([ - 'PLATFORM=gpu', - 'TVM_NUM_SHARDS=2', - 'TVM_SHARD_INDEX=1'], { - sh( + } else { + Utils.markStageSkippedForConditional('unittest: GPU 1 of 3') + } +} + +def shard_run_unittest_GPU_2_of_3() { + if (!skip_ci && is_docs_only_build != 1) { + node('GPU') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-gpu") { + try { + init_git() + timeout(time: max_time, unit: 'MINUTES') { + withEnv([ + 'PLATFORM=gpu', + 'TVM_NUM_SHARDS=3', + 'TVM_SHARD_INDEX=1'], { + sh( script: """ set -eux aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so @@ -821,86 +834,91 @@ stage('Test') { label: 'Download artifacts from S3', ) - ci_setup(ci_gpu) - sh ( - script: "${docker_run} ${ci_gpu} ./tests/scripts/task_java_unittest.sh", - label: 'Run Java unit tests', - ) - sh ( - script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_unittest_gpuonly.sh", - label: 'Run Python GPU unit tests', - ) - sh ( - script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_integration_gpuonly.sh", - label: 'Run Python GPU integration tests', - ) - }) - } - } finally { - junit 'build/pytest-results/*.xml' + ci_setup(ci_gpu) + sh ( + script: "${docker_run} ${ci_gpu} ./tests/scripts/task_java_unittest.sh", + label: 'Run Java unit tests', + ) + sh ( + script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_unittest_gpuonly.sh", + label: 'Run Python GPU unit tests', + ) + sh ( + script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_integration_gpuonly.sh", + label: 'Run Python GPU integration tests', + ) + }) } + } finally { + junit 'build/pytest-results/*.xml' } } - } else { - Utils.markStageSkippedForConditional('unittest: GPU 2 of 2') } - }, - 'integration: CPU 1 of 2': { - if (!skip_ci && is_docs_only_build != 1) { - node('CPU') { - ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-cpu") { - try { - init_git() - timeout(time: max_time, unit: 'MINUTES') { - withEnv([ - 'PLATFORM=cpu', - 'TVM_NUM_SHARDS=2', - 'TVM_SHARD_INDEX=0'], { - sh( + } else { + Utils.markStageSkippedForConditional('unittest: GPU 2 of 3') + } +} + +def shard_run_unittest_GPU_3_of_3() { + if (!skip_ci && is_docs_only_build != 1) { + node('GPU') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-gpu") { + try { + init_git() + timeout(time: max_time, unit: 'MINUTES') { + withEnv([ + 'PLATFORM=gpu', + 'TVM_NUM_SHARDS=3', + 'TVM_SHARD_INDEX=2'], { + sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_tsim.so build/libvta_tsim.so - md5sum build/libvta_tsim.so - aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm.so build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_fsim.so build/libvta_fsim.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm_runtime.so build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/config.cmake build/config.cmake + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake md5sum build/config.cmake """, label: 'Download artifacts from S3', ) - ci_setup(ci_cpu) - sh ( - script: "${docker_run} ${ci_cpu} ./tests/scripts/task_python_integration.sh", - label: 'Run CPU integration tests', - ) - }) - } - } finally { - junit 'build/pytest-results/*.xml' + ci_setup(ci_gpu) + sh ( + script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_unittest_gpuonly.sh", + label: 'Run Python GPU unit tests', + ) + sh ( + script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_integration_gpuonly.sh", + label: 'Run Python GPU integration tests', + ) + }) } + } finally { + junit 'build/pytest-results/*.xml' } } - } else { - Utils.markStageSkippedForConditional('integration: CPU 1 of 2') } - }, - 'integration: CPU 2 of 2': { - if (!skip_ci && is_docs_only_build != 1) { - node('CPU') { - ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-cpu") { - try { - init_git() - timeout(time: max_time, unit: 'MINUTES') { - withEnv([ - 'PLATFORM=cpu', - 'TVM_NUM_SHARDS=2', - 'TVM_SHARD_INDEX=1'], { - sh( + } else { + Utils.markStageSkippedForConditional('unittest: GPU 3 of 3') + } +} + + +def shard_run_integration_CPU_1_of_6() { + if (!skip_ci && is_docs_only_build != 1) { + node('CPU-SMALL') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-cpu") { + try { + init_git() + timeout(time: max_time, unit: 'MINUTES') { + withEnv([ + 'PLATFORM=cpu', + 'TVM_NUM_SHARDS=6', + 'TVM_SHARD_INDEX=0'], { + sh( script: """ set -eux aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_tsim.so build/libvta_tsim.so @@ -917,31 +935,35 @@ stage('Test') { label: 'Download artifacts from S3', ) - ci_setup(ci_cpu) - sh ( - script: "${docker_run} ${ci_cpu} ./tests/scripts/task_python_integration.sh", - label: 'Run CPU integration tests', - ) - }) - } - } finally { - junit 'build/pytest-results/*.xml' + ci_setup(ci_cpu) + sh ( + script: "${docker_run} ${ci_cpu} ./tests/scripts/task_python_integration.sh", + label: 'Run CPU integration tests', + ) + }) } + } finally { + junit 'build/pytest-results/*.xml' } } - } else { - Utils.markStageSkippedForConditional('integration: CPU 2 of 2') } - }, - 'unittest: CPU': { - if (!skip_ci && is_docs_only_build != 1) { - node('CPU-SMALL') { - ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-cpu") { + } else { + Utils.markStageSkippedForConditional('integration: CPU 1 of 6') + } +} + +def shard_run_integration_CPU_2_of_6() { + if (!skip_ci && is_docs_only_build != 1) { + node('CPU-SMALL') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-cpu") { + try { + init_git() timeout(time: max_time, unit: 'MINUTES') { - try { - init_git() - withEnv(['PLATFORM=cpu'], { - sh( + withEnv([ + 'PLATFORM=cpu', + 'TVM_NUM_SHARDS=6', + 'TVM_SHARD_INDEX=1'], { + sh( script: """ set -eux aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_tsim.so build/libvta_tsim.so @@ -958,259 +980,443 @@ stage('Test') { label: 'Download artifacts from S3', ) - ci_setup(ci_cpu) - cpp_unittest(ci_cpu) - python_unittest(ci_cpu) - fsim_test(ci_cpu) - sh ( - script: "${docker_run} ${ci_cpu} ./tests/scripts/task_python_vta_tsim.sh", - label: 'Run VTA tests in TSIM', - ) - }) - } finally { - junit 'build/pytest-results/*.xml' - } + ci_setup(ci_cpu) + sh ( + script: "${docker_run} ${ci_cpu} ./tests/scripts/task_python_integration.sh", + label: 'Run CPU integration tests', + ) + }) } + } finally { + junit 'build/pytest-results/*.xml' } } - } else { - Utils.markStageSkippedForConditional('unittest: CPU') } - }, - 'python: i386 1 of 3': { - if (!skip_ci && is_docs_only_build != 1) { - node('CPU-SMALL') { - ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-i386") { - try { - init_git() - timeout(time: max_time, unit: 'MINUTES') { - withEnv([ - 'PLATFORM=i386', - 'TVM_NUM_SHARDS=3', - 'TVM_SHARD_INDEX=0'], { - sh( + } else { + Utils.markStageSkippedForConditional('integration: CPU 2 of 6') + } +} + +def shard_run_integration_CPU_3_of_6() { + if (!skip_ci && is_docs_only_build != 1) { + node('CPU-SMALL') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-cpu") { + try { + init_git() + timeout(time: max_time, unit: 'MINUTES') { + withEnv([ + 'PLATFORM=cpu', + 'TVM_NUM_SHARDS=6', + 'TVM_SHARD_INDEX=2'], { + sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm.so build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_tsim.so build/libvta_tsim.so + md5sum build/libvta_tsim.so + aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libvta_fsim.so build/libvta_fsim.so + aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_fsim.so build/libvta_fsim.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm_runtime.so build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/i386/build/config.cmake build/config.cmake + aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/config.cmake build/config.cmake md5sum build/config.cmake """, label: 'Download artifacts from S3', ) - ci_setup(ci_i386) - cpp_unittest(ci_i386) - python_unittest(ci_i386) - sh ( - script: "${docker_run} ${ci_i386} ./tests/scripts/task_python_integration_i386only.sh", - label: 'Run i386 integration tests', - ) - fsim_test(ci_i386) - }) - } - } finally { - junit 'build/pytest-results/*.xml' + ci_setup(ci_cpu) + sh ( + script: "${docker_run} ${ci_cpu} ./tests/scripts/task_python_integration.sh", + label: 'Run CPU integration tests', + ) + }) } + } finally { + junit 'build/pytest-results/*.xml' } } - } else { - Utils.markStageSkippedForConditional('python: i386 1 of 3') } - }, - 'python: i386 2 of 3': { - if (!skip_ci && is_docs_only_build != 1) { - node('CPU-SMALL') { - ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-i386") { - try { - init_git() - timeout(time: max_time, unit: 'MINUTES') { - withEnv([ - 'PLATFORM=i386', - 'TVM_NUM_SHARDS=3', - 'TVM_SHARD_INDEX=1'], { - sh( + } else { + Utils.markStageSkippedForConditional('integration: CPU 3 of 6') + } +} + +def shard_run_integration_CPU_4_of_6() { + if (!skip_ci && is_docs_only_build != 1) { + node('CPU-SMALL') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-cpu") { + try { + init_git() + timeout(time: max_time, unit: 'MINUTES') { + withEnv([ + 'PLATFORM=cpu', + 'TVM_NUM_SHARDS=6', + 'TVM_SHARD_INDEX=3'], { + sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm.so build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_tsim.so build/libvta_tsim.so + md5sum build/libvta_tsim.so + aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libvta_fsim.so build/libvta_fsim.so + aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_fsim.so build/libvta_fsim.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm_runtime.so build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/i386/build/config.cmake build/config.cmake + aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/config.cmake build/config.cmake md5sum build/config.cmake """, label: 'Download artifacts from S3', ) - ci_setup(ci_i386) - python_unittest(ci_i386) - sh ( - script: "${docker_run} ${ci_i386} ./tests/scripts/task_python_integration_i386only.sh", - label: 'Run i386 integration tests', - ) - fsim_test(ci_i386) - }) - } - } finally { - junit 'build/pytest-results/*.xml' + ci_setup(ci_cpu) + sh ( + script: "${docker_run} ${ci_cpu} ./tests/scripts/task_python_integration.sh", + label: 'Run CPU integration tests', + ) + }) } + } finally { + junit 'build/pytest-results/*.xml' } } - } else { - Utils.markStageSkippedForConditional('python: i386 2 of 3') } - }, - 'python: i386 3 of 3': { - if (!skip_ci && is_docs_only_build != 1) { - node('CPU-SMALL') { - ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-i386") { - try { - init_git() - timeout(time: max_time, unit: 'MINUTES') { - withEnv([ - 'PLATFORM=i386', - 'TVM_NUM_SHARDS=3', - 'TVM_SHARD_INDEX=2'], { - sh( + } else { + Utils.markStageSkippedForConditional('integration: CPU 4 of 6') + } +} + +def shard_run_integration_CPU_5_of_6() { + if (!skip_ci && is_docs_only_build != 1) { + node('CPU-SMALL') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-cpu") { + try { + init_git() + timeout(time: max_time, unit: 'MINUTES') { + withEnv([ + 'PLATFORM=cpu', + 'TVM_NUM_SHARDS=6', + 'TVM_SHARD_INDEX=4'], { + sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm.so build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_tsim.so build/libvta_tsim.so + md5sum build/libvta_tsim.so + aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libvta_fsim.so build/libvta_fsim.so + aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_fsim.so build/libvta_fsim.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm_runtime.so build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/i386/build/config.cmake build/config.cmake + aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/config.cmake build/config.cmake md5sum build/config.cmake """, label: 'Download artifacts from S3', ) - ci_setup(ci_i386) - python_unittest(ci_i386) - sh ( - script: "${docker_run} ${ci_i386} ./tests/scripts/task_python_integration_i386only.sh", - label: 'Run i386 integration tests', - ) - fsim_test(ci_i386) - }) - } - } finally { - junit 'build/pytest-results/*.xml' + ci_setup(ci_cpu) + sh ( + script: "${docker_run} ${ci_cpu} ./tests/scripts/task_python_integration.sh", + label: 'Run CPU integration tests', + ) + }) } + } finally { + junit 'build/pytest-results/*.xml' } } - } else { - Utils.markStageSkippedForConditional('python: i386 3 of 3') } - }, - 'test: Hexagon 1 of 4': { - if (!skip_ci && is_docs_only_build != 1) { - node('CPU-SMALL') { - ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") { - try { - init_git() - timeout(time: max_time, unit: 'MINUTES') { - withEnv([ - 'PLATFORM=hexagon', - 'TVM_NUM_SHARDS=4', - 'TVM_SHARD_INDEX=0'], { - sh( + } else { + Utils.markStageSkippedForConditional('integration: CPU 5 of 6') + } +} + +def shard_run_integration_CPU_6_of_6() { + if (!skip_ci && is_docs_only_build != 1) { + node('CPU-SMALL') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-cpu") { + try { + init_git() + timeout(time: max_time, unit: 'MINUTES') { + withEnv([ + 'PLATFORM=cpu', + 'TVM_NUM_SHARDS=6', + 'TVM_SHARD_INDEX=5'], { + sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_tsim.so build/libvta_tsim.so + md5sum build/libvta_tsim.so + aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm_runtime.so build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_fsim.so build/libvta_fsim.so + md5sum build/libvta_fsim.so + aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/config.cmake build/config.cmake + aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/config.cmake build/config.cmake md5sum build/config.cmake """, label: 'Download artifacts from S3', ) - ci_setup(ci_hexagon) - cpp_unittest(ci_hexagon) - sh ( - script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_build_hexagon_api.sh", - label: 'Build Hexagon API', - ) - sh ( - script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_python_hexagon.sh", - label: 'Run Hexagon tests', - ) - }) - } - } finally { - junit 'build/pytest-results/*.xml' + ci_setup(ci_cpu) + sh ( + script: "${docker_run} ${ci_cpu} ./tests/scripts/task_python_integration.sh", + label: 'Run CPU integration tests', + ) + }) } + } finally { + junit 'build/pytest-results/*.xml' } } - } else { - Utils.markStageSkippedForConditional('test: Hexagon 1 of 4') } - }, - 'test: Hexagon 2 of 4': { - if (!skip_ci && is_docs_only_build != 1) { - node('CPU-SMALL') { - ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") { - try { - init_git() - timeout(time: max_time, unit: 'MINUTES') { - withEnv([ - 'PLATFORM=hexagon', - 'TVM_NUM_SHARDS=4', - 'TVM_SHARD_INDEX=1'], { - sh( + } else { + Utils.markStageSkippedForConditional('integration: CPU 6 of 6') + } +} + + +def shard_run_python_i386_1_of_5() { + if (!skip_ci && is_docs_only_build != 1) { + node('CPU-SMALL') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-i386") { + try { + init_git() + timeout(time: max_time, unit: 'MINUTES') { + withEnv([ + 'PLATFORM=i386', + 'TVM_NUM_SHARDS=5', + 'TVM_SHARD_INDEX=0'], { + sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm_runtime.so build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libvta_fsim.so build/libvta_fsim.so + md5sum build/libvta_fsim.so + aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/config.cmake build/config.cmake + aws s3 cp --no-progress s3://${s3_prefix}/i386/build/config.cmake build/config.cmake md5sum build/config.cmake """, label: 'Download artifacts from S3', ) - ci_setup(ci_hexagon) - sh ( - script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_build_hexagon_api.sh", - label: 'Build Hexagon API', - ) - sh ( - script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_python_hexagon.sh", - label: 'Run Hexagon tests', - ) - }) - } - } finally { - junit 'build/pytest-results/*.xml' + ci_setup(ci_i386) + cpp_unittest(ci_i386) + python_unittest(ci_i386) + sh ( + script: "${docker_run} ${ci_i386} ./tests/scripts/task_python_integration_i386only.sh", + label: 'Run i386 integration tests', + ) + fsim_test(ci_i386) + }) } + } finally { + junit 'build/pytest-results/*.xml' } } - } else { - Utils.markStageSkippedForConditional('test: Hexagon 2 of 4') } - }, - 'test: Hexagon 3 of 4': { - if (!skip_ci && is_docs_only_build != 1) { - node('CPU-SMALL') { - ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") { - try { - init_git() - timeout(time: max_time, unit: 'MINUTES') { - withEnv([ - 'PLATFORM=hexagon', - 'TVM_NUM_SHARDS=4', - 'TVM_SHARD_INDEX=2'], { - sh( + } else { + Utils.markStageSkippedForConditional('python: i386 1 of 5') + } +} + +def shard_run_python_i386_2_of_5() { + if (!skip_ci && is_docs_only_build != 1) { + node('CPU-SMALL') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-i386") { + try { + init_git() + timeout(time: max_time, unit: 'MINUTES') { + withEnv([ + 'PLATFORM=i386', + 'TVM_NUM_SHARDS=5', + 'TVM_SHARD_INDEX=1'], { + sh( + script: """ + set -eux + aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm.so build/libtvm.so + md5sum build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libvta_fsim.so build/libvta_fsim.so + md5sum build/libvta_fsim.so + aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm_runtime.so build/libtvm_runtime.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/i386/build/config.cmake build/config.cmake + md5sum build/config.cmake + """, + label: 'Download artifacts from S3', + ) + + ci_setup(ci_i386) + python_unittest(ci_i386) + sh ( + script: "${docker_run} ${ci_i386} ./tests/scripts/task_python_integration_i386only.sh", + label: 'Run i386 integration tests', + ) + fsim_test(ci_i386) + }) + } + } finally { + junit 'build/pytest-results/*.xml' + } + } + } + } else { + Utils.markStageSkippedForConditional('python: i386 2 of 5') + } +} + +def shard_run_python_i386_3_of_5() { + if (!skip_ci && is_docs_only_build != 1) { + node('CPU-SMALL') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-i386") { + try { + init_git() + timeout(time: max_time, unit: 'MINUTES') { + withEnv([ + 'PLATFORM=i386', + 'TVM_NUM_SHARDS=5', + 'TVM_SHARD_INDEX=2'], { + sh( + script: """ + set -eux + aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm.so build/libtvm.so + md5sum build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libvta_fsim.so build/libvta_fsim.so + md5sum build/libvta_fsim.so + aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm_runtime.so build/libtvm_runtime.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/i386/build/config.cmake build/config.cmake + md5sum build/config.cmake + """, + label: 'Download artifacts from S3', + ) + + ci_setup(ci_i386) + python_unittest(ci_i386) + sh ( + script: "${docker_run} ${ci_i386} ./tests/scripts/task_python_integration_i386only.sh", + label: 'Run i386 integration tests', + ) + fsim_test(ci_i386) + }) + } + } finally { + junit 'build/pytest-results/*.xml' + } + } + } + } else { + Utils.markStageSkippedForConditional('python: i386 3 of 5') + } +} + +def shard_run_python_i386_4_of_5() { + if (!skip_ci && is_docs_only_build != 1) { + node('CPU-SMALL') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-i386") { + try { + init_git() + timeout(time: max_time, unit: 'MINUTES') { + withEnv([ + 'PLATFORM=i386', + 'TVM_NUM_SHARDS=5', + 'TVM_SHARD_INDEX=3'], { + sh( + script: """ + set -eux + aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm.so build/libtvm.so + md5sum build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libvta_fsim.so build/libvta_fsim.so + md5sum build/libvta_fsim.so + aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm_runtime.so build/libtvm_runtime.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/i386/build/config.cmake build/config.cmake + md5sum build/config.cmake + """, + label: 'Download artifacts from S3', + ) + + ci_setup(ci_i386) + python_unittest(ci_i386) + sh ( + script: "${docker_run} ${ci_i386} ./tests/scripts/task_python_integration_i386only.sh", + label: 'Run i386 integration tests', + ) + fsim_test(ci_i386) + }) + } + } finally { + junit 'build/pytest-results/*.xml' + } + } + } + } else { + Utils.markStageSkippedForConditional('python: i386 4 of 5') + } +} + +def shard_run_python_i386_5_of_5() { + if (!skip_ci && is_docs_only_build != 1) { + node('CPU-SMALL') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-i386") { + try { + init_git() + timeout(time: max_time, unit: 'MINUTES') { + withEnv([ + 'PLATFORM=i386', + 'TVM_NUM_SHARDS=5', + 'TVM_SHARD_INDEX=4'], { + sh( + script: """ + set -eux + aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm.so build/libtvm.so + md5sum build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libvta_fsim.so build/libvta_fsim.so + md5sum build/libvta_fsim.so + aws s3 cp --no-progress s3://${s3_prefix}/i386/build/libtvm_runtime.so build/libtvm_runtime.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/i386/build/config.cmake build/config.cmake + md5sum build/config.cmake + """, + label: 'Download artifacts from S3', + ) + + ci_setup(ci_i386) + python_unittest(ci_i386) + sh ( + script: "${docker_run} ${ci_i386} ./tests/scripts/task_python_integration_i386only.sh", + label: 'Run i386 integration tests', + ) + fsim_test(ci_i386) + }) + } + } finally { + junit 'build/pytest-results/*.xml' + } + } + } + } else { + Utils.markStageSkippedForConditional('python: i386 5 of 5') + } +} + + +def shard_run_test_Hexagon_1_of_7() { + if (!skip_ci && is_docs_only_build != 1) { + node('CPU-SMALL') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") { + try { + init_git() + timeout(time: max_time, unit: 'MINUTES') { + withEnv([ + 'PLATFORM=hexagon', + 'TVM_NUM_SHARDS=7', + 'TVM_SHARD_INDEX=0'], { + sh( script: """ set -eux aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so @@ -1219,42 +1425,42 @@ stage('Test') { md5sum build/libtvm_runtime.so aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/config.cmake build/config.cmake md5sum build/config.cmake + aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/hexagon_api_output build/hexagon_api_output --recursive """, label: 'Download artifacts from S3', ) - ci_setup(ci_hexagon) - sh ( - script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_build_hexagon_api.sh", - label: 'Build Hexagon API', - ) - sh ( - script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_python_hexagon.sh", - label: 'Run Hexagon tests', - ) - }) - } - } finally { - junit 'build/pytest-results/*.xml' + add_hexagon_permissions() + ci_setup(ci_hexagon) + cpp_unittest(ci_hexagon) + sh ( + script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_python_hexagon.sh", + label: 'Run Hexagon tests', + ) + }) } + } finally { + junit 'build/pytest-results/*.xml' } } - } else { - Utils.markStageSkippedForConditional('test: Hexagon 3 of 4') } - }, - 'test: Hexagon 4 of 4': { - if (!skip_ci && is_docs_only_build != 1) { - node('CPU-SMALL') { - ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") { - try { - init_git() - timeout(time: max_time, unit: 'MINUTES') { - withEnv([ - 'PLATFORM=hexagon', - 'TVM_NUM_SHARDS=4', - 'TVM_SHARD_INDEX=3'], { - sh( + } else { + Utils.markStageSkippedForConditional('test: Hexagon 1 of 7') + } +} + +def shard_run_test_Hexagon_2_of_7() { + if (!skip_ci && is_docs_only_build != 1) { + node('CPU-SMALL') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") { + try { + init_git() + timeout(time: max_time, unit: 'MINUTES') { + withEnv([ + 'PLATFORM=hexagon', + 'TVM_NUM_SHARDS=7', + 'TVM_SHARD_INDEX=1'], { + sh( script: """ set -eux aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so @@ -1263,173 +1469,866 @@ stage('Test') { md5sum build/libtvm_runtime.so aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/config.cmake build/config.cmake md5sum build/config.cmake + aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/hexagon_api_output build/hexagon_api_output --recursive """, label: 'Download artifacts from S3', ) - ci_setup(ci_hexagon) - sh ( - script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_build_hexagon_api.sh", - label: 'Build Hexagon API', - ) - sh ( - script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_python_hexagon.sh", - label: 'Run Hexagon tests', - ) - }) - } - } finally { - junit 'build/pytest-results/*.xml' + add_hexagon_permissions() + ci_setup(ci_hexagon) + sh ( + script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_python_hexagon.sh", + label: 'Run Hexagon tests', + ) + }) } + } finally { + junit 'build/pytest-results/*.xml' } } - } else { - Utils.markStageSkippedForConditional('test: Hexagon 4 of 4') } - }, - 'test: QEMU': { - if (!skip_ci && is_docs_only_build != 1) { - node('CPU-SMALL') { - ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-qemu") { + } else { + Utils.markStageSkippedForConditional('test: Hexagon 2 of 7') + } +} + +def shard_run_test_Hexagon_3_of_7() { + if (!skip_ci && is_docs_only_build != 1) { + node('CPU-SMALL') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") { + try { + init_git() timeout(time: max_time, unit: 'MINUTES') { - try { - init_git() - withEnv(['PLATFORM=qemu'], { - sh( + withEnv([ + 'PLATFORM=hexagon', + 'TVM_NUM_SHARDS=7', + 'TVM_SHARD_INDEX=2'], { + sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/qemu/build/libtvm.so build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/qemu/build/libtvm_runtime.so build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/qemu/build/config.cmake build/config.cmake + aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/config.cmake build/config.cmake md5sum build/config.cmake - aws s3 cp --no-progress s3://${s3_prefix}/qemu/build/microtvm_template_projects build/microtvm_template_projects --recursive + aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/hexagon_api_output build/hexagon_api_output --recursive """, label: 'Download artifacts from S3', ) - add_microtvm_permissions() - ci_setup(ci_qemu) - cpp_unittest(ci_qemu) - sh ( - script: "${docker_run} ${ci_qemu} ./tests/scripts/task_python_microtvm.sh", - label: 'Run microTVM tests', - ) - sh ( - script: "${docker_run} ${ci_qemu} ./tests/scripts/task_demo_microtvm.sh", - label: 'Run microTVM demos', - ) - }) - } finally { - junit 'build/pytest-results/*.xml' - } + add_hexagon_permissions() + ci_setup(ci_hexagon) + sh ( + script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_python_hexagon.sh", + label: 'Run Hexagon tests', + ) + }) } + } finally { + junit 'build/pytest-results/*.xml' } } - } else { - Utils.markStageSkippedForConditional('test: QEMU') } - }, - 'topi: aarch64': { - if (!skip_ci && is_docs_only_build != 1) { - node('ARM') { - ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") { + } else { + Utils.markStageSkippedForConditional('test: Hexagon 3 of 7') + } +} + +def shard_run_test_Hexagon_4_of_7() { + if (!skip_ci && is_docs_only_build != 1) { + node('CPU-SMALL') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") { + try { + init_git() timeout(time: max_time, unit: 'MINUTES') { - try { - init_git() - withEnv(['PLATFORM=arm'], { - sh( + withEnv([ + 'PLATFORM=hexagon', + 'TVM_NUM_SHARDS=7', + 'TVM_SHARD_INDEX=3'], { + sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libvta_fsim.so build/libvta_fsim.so - md5sum build/libvta_fsim.so - aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm_runtime.so build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/arm/build/config.cmake build/config.cmake + aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/config.cmake build/config.cmake md5sum build/config.cmake + aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/hexagon_api_output build/hexagon_api_output --recursive """, label: 'Download artifacts from S3', ) - ci_setup(ci_arm) - cpp_unittest(ci_arm) - sh ( - script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_arm_compute_library.sh", - label: 'Run test_arm_compute_lib test', - ) - sh ( - script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_topi.sh", - label: 'Run TOPI tests', - ) - }) - } finally { - junit 'build/pytest-results/*.xml' - } + add_hexagon_permissions() + ci_setup(ci_hexagon) + sh ( + script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_python_hexagon.sh", + label: 'Run Hexagon tests', + ) + }) + } + } finally { + junit 'build/pytest-results/*.xml' + } + } + } + } else { + Utils.markStageSkippedForConditional('test: Hexagon 4 of 7') + } +} + +def shard_run_test_Hexagon_5_of_7() { + if (!skip_ci && is_docs_only_build != 1) { + node('CPU-SMALL') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") { + try { + init_git() + timeout(time: max_time, unit: 'MINUTES') { + withEnv([ + 'PLATFORM=hexagon', + 'TVM_NUM_SHARDS=7', + 'TVM_SHARD_INDEX=4'], { + sh( + script: """ + set -eux + aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so + md5sum build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm_runtime.so build/libtvm_runtime.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/config.cmake build/config.cmake + md5sum build/config.cmake + aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/hexagon_api_output build/hexagon_api_output --recursive + """, + label: 'Download artifacts from S3', + ) + + add_hexagon_permissions() + ci_setup(ci_hexagon) + sh ( + script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_python_hexagon.sh", + label: 'Run Hexagon tests', + ) + }) + } + } finally { + junit 'build/pytest-results/*.xml' + } + } + } + } else { + Utils.markStageSkippedForConditional('test: Hexagon 5 of 7') + } +} + +def shard_run_test_Hexagon_6_of_7() { + if (!skip_ci && is_docs_only_build != 1) { + node('CPU-SMALL') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") { + try { + init_git() + timeout(time: max_time, unit: 'MINUTES') { + withEnv([ + 'PLATFORM=hexagon', + 'TVM_NUM_SHARDS=7', + 'TVM_SHARD_INDEX=5'], { + sh( + script: """ + set -eux + aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so + md5sum build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm_runtime.so build/libtvm_runtime.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/config.cmake build/config.cmake + md5sum build/config.cmake + aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/hexagon_api_output build/hexagon_api_output --recursive + """, + label: 'Download artifacts from S3', + ) + + add_hexagon_permissions() + ci_setup(ci_hexagon) + sh ( + script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_python_hexagon.sh", + label: 'Run Hexagon tests', + ) + }) + } + } finally { + junit 'build/pytest-results/*.xml' + } + } + } + } else { + Utils.markStageSkippedForConditional('test: Hexagon 6 of 7') + } +} + +def shard_run_test_Hexagon_7_of_7() { + if (!skip_ci && is_docs_only_build != 1) { + node('CPU-SMALL') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") { + try { + init_git() + timeout(time: max_time, unit: 'MINUTES') { + withEnv([ + 'PLATFORM=hexagon', + 'TVM_NUM_SHARDS=7', + 'TVM_SHARD_INDEX=6'], { + sh( + script: """ + set -eux + aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm.so build/libtvm.so + md5sum build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/libtvm_runtime.so build/libtvm_runtime.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/config.cmake build/config.cmake + md5sum build/config.cmake + aws s3 cp --no-progress s3://${s3_prefix}/hexagon/build/hexagon_api_output build/hexagon_api_output --recursive + """, + label: 'Download artifacts from S3', + ) + + add_hexagon_permissions() + ci_setup(ci_hexagon) + sh ( + script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_python_hexagon.sh", + label: 'Run Hexagon tests', + ) + }) + } + } finally { + junit 'build/pytest-results/*.xml' + } + } + } + } else { + Utils.markStageSkippedForConditional('test: Hexagon 7 of 7') + } +} + + +def shard_run_integration_aarch64_1_of_4() { + if (!skip_ci && is_docs_only_build != 1) { + node('ARM') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") { + try { + init_git() + timeout(time: max_time, unit: 'MINUTES') { + withEnv([ + 'PLATFORM=arm', + 'TVM_NUM_SHARDS=4', + 'TVM_SHARD_INDEX=0'], { + sh( + script: """ + set -eux + aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so + md5sum build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libvta_fsim.so build/libvta_fsim.so + md5sum build/libvta_fsim.so + aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm_runtime.so build/libtvm_runtime.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/arm/build/config.cmake build/config.cmake + md5sum build/config.cmake + """, + label: 'Download artifacts from S3', + ) + + ci_setup(ci_arm) + python_unittest(ci_arm) + sh ( + script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_integration.sh", + label: 'Run CPU integration tests', + ) + }) + } + } finally { + junit 'build/pytest-results/*.xml' + } + } + } + } else { + Utils.markStageSkippedForConditional('integration: aarch64 1 of 4') + } +} + +def shard_run_integration_aarch64_2_of_4() { + if (!skip_ci && is_docs_only_build != 1) { + node('ARM') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") { + try { + init_git() + timeout(time: max_time, unit: 'MINUTES') { + withEnv([ + 'PLATFORM=arm', + 'TVM_NUM_SHARDS=4', + 'TVM_SHARD_INDEX=1'], { + sh( + script: """ + set -eux + aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so + md5sum build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libvta_fsim.so build/libvta_fsim.so + md5sum build/libvta_fsim.so + aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm_runtime.so build/libtvm_runtime.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/arm/build/config.cmake build/config.cmake + md5sum build/config.cmake + """, + label: 'Download artifacts from S3', + ) + + ci_setup(ci_arm) + python_unittest(ci_arm) + sh ( + script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_integration.sh", + label: 'Run CPU integration tests', + ) + }) + } + } finally { + junit 'build/pytest-results/*.xml' + } + } + } + } else { + Utils.markStageSkippedForConditional('integration: aarch64 2 of 4') + } +} + +def shard_run_integration_aarch64_3_of_4() { + if (!skip_ci && is_docs_only_build != 1) { + node('ARM') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") { + try { + init_git() + timeout(time: max_time, unit: 'MINUTES') { + withEnv([ + 'PLATFORM=arm', + 'TVM_NUM_SHARDS=4', + 'TVM_SHARD_INDEX=2'], { + sh( + script: """ + set -eux + aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so + md5sum build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libvta_fsim.so build/libvta_fsim.so + md5sum build/libvta_fsim.so + aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm_runtime.so build/libtvm_runtime.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/arm/build/config.cmake build/config.cmake + md5sum build/config.cmake + """, + label: 'Download artifacts from S3', + ) + + ci_setup(ci_arm) + python_unittest(ci_arm) + sh ( + script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_integration.sh", + label: 'Run CPU integration tests', + ) + }) + } + } finally { + junit 'build/pytest-results/*.xml' + } + } + } + } else { + Utils.markStageSkippedForConditional('integration: aarch64 3 of 4') + } +} + +def shard_run_integration_aarch64_4_of_4() { + if (!skip_ci && is_docs_only_build != 1) { + node('ARM') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") { + try { + init_git() + timeout(time: max_time, unit: 'MINUTES') { + withEnv([ + 'PLATFORM=arm', + 'TVM_NUM_SHARDS=4', + 'TVM_SHARD_INDEX=3'], { + sh( + script: """ + set -eux + aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so + md5sum build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libvta_fsim.so build/libvta_fsim.so + md5sum build/libvta_fsim.so + aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm_runtime.so build/libtvm_runtime.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/arm/build/config.cmake build/config.cmake + md5sum build/config.cmake + """, + label: 'Download artifacts from S3', + ) + + ci_setup(ci_arm) + python_unittest(ci_arm) + sh ( + script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_integration.sh", + label: 'Run CPU integration tests', + ) + }) + } + } finally { + junit 'build/pytest-results/*.xml' + } + } + } + } else { + Utils.markStageSkippedForConditional('integration: aarch64 4 of 4') + } +} + + +def shard_run_topi_GPU_1_of_4() { + if (!skip_ci && is_docs_only_build != 1) { + node('GPU') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/topi-python-gpu") { + try { + init_git() + timeout(time: max_time, unit: 'MINUTES') { + withEnv([ + 'PLATFORM=gpu', + 'TVM_NUM_SHARDS=4', + 'TVM_SHARD_INDEX=0'], { + sh( + script: """ + set -eux + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so + md5sum build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so + md5sum build/libvta_fsim.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake + md5sum build/config.cmake + """, + label: 'Download artifacts from S3', + ) + + ci_setup(ci_gpu) + sh ( + script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_topi.sh", + label: 'Run TOPI tests', + ) + }) + } + } finally { + junit 'build/pytest-results/*.xml' + } + } + } + } else { + Utils.markStageSkippedForConditional('topi: GPU 1 of 4') + } +} + +def shard_run_topi_GPU_2_of_4() { + if (!skip_ci && is_docs_only_build != 1) { + node('GPU') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/topi-python-gpu") { + try { + init_git() + timeout(time: max_time, unit: 'MINUTES') { + withEnv([ + 'PLATFORM=gpu', + 'TVM_NUM_SHARDS=4', + 'TVM_SHARD_INDEX=1'], { + sh( + script: """ + set -eux + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so + md5sum build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so + md5sum build/libvta_fsim.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake + md5sum build/config.cmake + """, + label: 'Download artifacts from S3', + ) + + ci_setup(ci_gpu) + sh ( + script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_topi.sh", + label: 'Run TOPI tests', + ) + }) + } + } finally { + junit 'build/pytest-results/*.xml' + } + } + } + } else { + Utils.markStageSkippedForConditional('topi: GPU 2 of 4') + } +} + +def shard_run_topi_GPU_3_of_4() { + if (!skip_ci && is_docs_only_build != 1) { + node('GPU') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/topi-python-gpu") { + try { + init_git() + timeout(time: max_time, unit: 'MINUTES') { + withEnv([ + 'PLATFORM=gpu', + 'TVM_NUM_SHARDS=4', + 'TVM_SHARD_INDEX=2'], { + sh( + script: """ + set -eux + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so + md5sum build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so + md5sum build/libvta_fsim.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake + md5sum build/config.cmake + """, + label: 'Download artifacts from S3', + ) + + ci_setup(ci_gpu) + sh ( + script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_topi.sh", + label: 'Run TOPI tests', + ) + }) + } + } finally { + junit 'build/pytest-results/*.xml' + } + } + } + } else { + Utils.markStageSkippedForConditional('topi: GPU 3 of 4') + } +} + +def shard_run_topi_GPU_4_of_4() { + if (!skip_ci && is_docs_only_build != 1) { + node('GPU') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/topi-python-gpu") { + try { + init_git() + timeout(time: max_time, unit: 'MINUTES') { + withEnv([ + 'PLATFORM=gpu', + 'TVM_NUM_SHARDS=4', + 'TVM_SHARD_INDEX=3'], { + sh( + script: """ + set -eux + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so + md5sum build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so + md5sum build/libvta_fsim.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake + md5sum build/config.cmake + """, + label: 'Download artifacts from S3', + ) + + ci_setup(ci_gpu) + sh ( + script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_topi.sh", + label: 'Run TOPI tests', + ) + }) + } + } finally { + junit 'build/pytest-results/*.xml' + } + } + } + } else { + Utils.markStageSkippedForConditional('topi: GPU 4 of 4') + } +} + + +def shard_run_frontend_GPU_1_of_6() { + if (!skip_ci && is_docs_only_build != 1) { + node('GPU') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") { + try { + init_git() + timeout(time: max_time, unit: 'MINUTES') { + withEnv([ + 'PLATFORM=gpu', + 'TVM_NUM_SHARDS=6', + 'TVM_SHARD_INDEX=0'], { + sh( + script: """ + set -eux + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so + md5sum build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so + md5sum build/libvta_fsim.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake + md5sum build/config.cmake + """, + label: 'Download artifacts from S3', + ) + + ci_setup(ci_gpu) + sh ( + script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_frontend.sh", + label: 'Run Python frontend tests', + ) + }) + } + } finally { + junit 'build/pytest-results/*.xml' + } + } + } + } else { + Utils.markStageSkippedForConditional('frontend: GPU 1 of 6') + } +} + +def shard_run_frontend_GPU_2_of_6() { + if (!skip_ci && is_docs_only_build != 1) { + node('GPU') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") { + try { + init_git() + timeout(time: max_time, unit: 'MINUTES') { + withEnv([ + 'PLATFORM=gpu', + 'TVM_NUM_SHARDS=6', + 'TVM_SHARD_INDEX=1'], { + sh( + script: """ + set -eux + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so + md5sum build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so + md5sum build/libvta_fsim.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake + md5sum build/config.cmake + """, + label: 'Download artifacts from S3', + ) + + ci_setup(ci_gpu) + sh ( + script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_frontend.sh", + label: 'Run Python frontend tests', + ) + }) + } + } finally { + junit 'build/pytest-results/*.xml' + } + } + } + } else { + Utils.markStageSkippedForConditional('frontend: GPU 2 of 6') + } +} + +def shard_run_frontend_GPU_3_of_6() { + if (!skip_ci && is_docs_only_build != 1) { + node('GPU') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") { + try { + init_git() + timeout(time: max_time, unit: 'MINUTES') { + withEnv([ + 'PLATFORM=gpu', + 'TVM_NUM_SHARDS=6', + 'TVM_SHARD_INDEX=2'], { + sh( + script: """ + set -eux + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so + md5sum build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so + md5sum build/libvta_fsim.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake + md5sum build/config.cmake + """, + label: 'Download artifacts from S3', + ) + + ci_setup(ci_gpu) + sh ( + script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_frontend.sh", + label: 'Run Python frontend tests', + ) + }) + } + } finally { + junit 'build/pytest-results/*.xml' + } + } + } + } else { + Utils.markStageSkippedForConditional('frontend: GPU 3 of 6') + } +} + +def shard_run_frontend_GPU_4_of_6() { + if (!skip_ci && is_docs_only_build != 1) { + node('GPU') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") { + try { + init_git() + timeout(time: max_time, unit: 'MINUTES') { + withEnv([ + 'PLATFORM=gpu', + 'TVM_NUM_SHARDS=6', + 'TVM_SHARD_INDEX=3'], { + sh( + script: """ + set -eux + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so + md5sum build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so + md5sum build/libvta_fsim.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake + md5sum build/config.cmake + """, + label: 'Download artifacts from S3', + ) + + ci_setup(ci_gpu) + sh ( + script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_frontend.sh", + label: 'Run Python frontend tests', + ) + }) + } + } finally { + junit 'build/pytest-results/*.xml' + } + } + } + } else { + Utils.markStageSkippedForConditional('frontend: GPU 4 of 6') + } +} + +def shard_run_frontend_GPU_5_of_6() { + if (!skip_ci && is_docs_only_build != 1) { + node('GPU') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") { + try { + init_git() + timeout(time: max_time, unit: 'MINUTES') { + withEnv([ + 'PLATFORM=gpu', + 'TVM_NUM_SHARDS=6', + 'TVM_SHARD_INDEX=4'], { + sh( + script: """ + set -eux + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so + md5sum build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so + md5sum build/libvta_fsim.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so + md5sum build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake + md5sum build/config.cmake + """, + label: 'Download artifacts from S3', + ) + + ci_setup(ci_gpu) + sh ( + script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_frontend.sh", + label: 'Run Python frontend tests', + ) + }) } + } finally { + junit 'build/pytest-results/*.xml' } } - } else { - Utils.markStageSkippedForConditional('topi: aarch64') } - }, - 'integration: aarch64 1 of 2': { - if (!skip_ci && is_docs_only_build != 1) { - node('ARM') { - ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") { - try { - init_git() - timeout(time: max_time, unit: 'MINUTES') { - withEnv([ - 'PLATFORM=arm', - 'TVM_NUM_SHARDS=2', - 'TVM_SHARD_INDEX=0'], { - sh( + } else { + Utils.markStageSkippedForConditional('frontend: GPU 5 of 6') + } +} + +def shard_run_frontend_GPU_6_of_6() { + if (!skip_ci && is_docs_only_build != 1) { + node('GPU') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") { + try { + init_git() + timeout(time: max_time, unit: 'MINUTES') { + withEnv([ + 'PLATFORM=gpu', + 'TVM_NUM_SHARDS=6', + 'TVM_SHARD_INDEX=5'], { + sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libvta_fsim.so build/libvta_fsim.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm_runtime.so build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/arm/build/config.cmake build/config.cmake + aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake md5sum build/config.cmake """, label: 'Download artifacts from S3', ) - ci_setup(ci_arm) - python_unittest(ci_arm) - sh ( - script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_integration.sh", - label: 'Run CPU integration tests', - ) - }) - } - } finally { - junit 'build/pytest-results/*.xml' + ci_setup(ci_gpu) + sh ( + script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_frontend.sh", + label: 'Run Python frontend tests', + ) + }) } + } finally { + junit 'build/pytest-results/*.xml' } } - } else { - Utils.markStageSkippedForConditional('integration: aarch64 1 of 2') } - }, - 'integration: aarch64 2 of 2': { - if (!skip_ci && is_docs_only_build != 1) { - node('ARM') { - ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") { - try { - init_git() - timeout(time: max_time, unit: 'MINUTES') { - withEnv([ - 'PLATFORM=arm', - 'TVM_NUM_SHARDS=2', - 'TVM_SHARD_INDEX=1'], { - sh( + } else { + Utils.markStageSkippedForConditional('frontend: GPU 6 of 6') + } +} + + +def shard_run_topi_aarch64_1_of_2() { + if (!skip_ci && is_docs_only_build != 1) { + node('ARM') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") { + try { + init_git() + timeout(time: max_time, unit: 'MINUTES') { + withEnv([ + 'PLATFORM=arm', + 'TVM_NUM_SHARDS=2', + 'TVM_SHARD_INDEX=0'], { + sh( script: """ set -eux aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so @@ -1444,236 +2343,379 @@ stage('Test') { label: 'Download artifacts from S3', ) - ci_setup(ci_arm) - python_unittest(ci_arm) - sh ( - script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_integration.sh", - label: 'Run CPU integration tests', - ) - }) - } - } finally { - junit 'build/pytest-results/*.xml' + ci_setup(ci_arm) + cpp_unittest(ci_arm) + sh ( + script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_arm_compute_library.sh", + label: 'Run test_arm_compute_lib test', + ) + sh ( + script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_topi.sh", + label: 'Run TOPI tests', + ) + }) } + } finally { + junit 'build/pytest-results/*.xml' } } - } else { - Utils.markStageSkippedForConditional('integration: aarch64 2 of 2') } - }, - 'topi: GPU 1 of 2': { - if (!skip_ci && is_docs_only_build != 1) { - node('GPU') { - ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/topi-python-gpu") { - try { - init_git() - timeout(time: max_time, unit: 'MINUTES') { - withEnv([ - 'PLATFORM=gpu', - 'TVM_NUM_SHARDS=2', - 'TVM_SHARD_INDEX=0'], { - sh( + } else { + Utils.markStageSkippedForConditional('topi: aarch64 1 of 2') + } +} + +def shard_run_topi_aarch64_2_of_2() { + if (!skip_ci && is_docs_only_build != 1) { + node('ARM') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") { + try { + init_git() + timeout(time: max_time, unit: 'MINUTES') { + withEnv([ + 'PLATFORM=arm', + 'TVM_NUM_SHARDS=2', + 'TVM_SHARD_INDEX=1'], { + sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so + aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libvta_fsim.so build/libvta_fsim.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake + aws s3 cp --no-progress s3://${s3_prefix}/arm/build/config.cmake build/config.cmake md5sum build/config.cmake """, label: 'Download artifacts from S3', ) - ci_setup(ci_gpu) - sh ( - script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_topi.sh", - label: 'Run TOPI tests', - ) - }) - } - } finally { - junit 'build/pytest-results/*.xml' + ci_setup(ci_arm) + cpp_unittest(ci_arm) + sh ( + script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_arm_compute_library.sh", + label: 'Run test_arm_compute_lib test', + ) + sh ( + script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_topi.sh", + label: 'Run TOPI tests', + ) + }) } + } finally { + junit 'build/pytest-results/*.xml' } } - } else { - Utils.markStageSkippedForConditional('topi: GPU 1 of 2') } - }, - 'topi: GPU 2 of 2': { - if (!skip_ci && is_docs_only_build != 1) { - node('GPU') { - ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/topi-python-gpu") { - try { - init_git() - timeout(time: max_time, unit: 'MINUTES') { - withEnv([ - 'PLATFORM=gpu', - 'TVM_NUM_SHARDS=2', - 'TVM_SHARD_INDEX=1'], { - sh( + } else { + Utils.markStageSkippedForConditional('topi: aarch64 2 of 2') + } +} + + +def shard_run_frontend_aarch64_1_of_2() { + if (!skip_ci && is_docs_only_build != 1) { + node('ARM') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-arm") { + try { + init_git() + timeout(time: max_time, unit: 'MINUTES') { + withEnv([ + 'PLATFORM=arm', + 'TVM_NUM_SHARDS=2', + 'TVM_SHARD_INDEX=0'], { + sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so + aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libvta_fsim.so build/libvta_fsim.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake + aws s3 cp --no-progress s3://${s3_prefix}/arm/build/config.cmake build/config.cmake md5sum build/config.cmake """, label: 'Download artifacts from S3', ) - ci_setup(ci_gpu) - sh ( - script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_topi.sh", - label: 'Run TOPI tests', - ) - }) - } - } finally { - junit 'build/pytest-results/*.xml' + ci_setup(ci_arm) + sh ( + script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_frontend_cpu.sh", + label: 'Run Python frontend tests', + ) + }) } + } finally { + junit 'build/pytest-results/*.xml' } } - } else { - Utils.markStageSkippedForConditional('topi: GPU 2 of 2') } - }, - 'frontend: GPU 1 of 3': { - if (!skip_ci && is_docs_only_build != 1) { - node('GPU') { - ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") { - try { - init_git() - timeout(time: max_time, unit: 'MINUTES') { - withEnv([ - 'PLATFORM=gpu', - 'TVM_NUM_SHARDS=3', - 'TVM_SHARD_INDEX=0'], { - sh( + } else { + Utils.markStageSkippedForConditional('frontend: aarch64 1 of 2') + } +} + +def shard_run_frontend_aarch64_2_of_2() { + if (!skip_ci && is_docs_only_build != 1) { + node('ARM') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-arm") { + try { + init_git() + timeout(time: max_time, unit: 'MINUTES') { + withEnv([ + 'PLATFORM=arm', + 'TVM_NUM_SHARDS=2', + 'TVM_SHARD_INDEX=1'], { + sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so + aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libvta_fsim.so build/libvta_fsim.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake + aws s3 cp --no-progress s3://${s3_prefix}/arm/build/config.cmake build/config.cmake md5sum build/config.cmake """, label: 'Download artifacts from S3', ) - ci_setup(ci_gpu) - sh ( - script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_frontend.sh", - label: 'Run Python frontend tests', - ) - }) - } - } finally { - junit 'build/pytest-results/*.xml' + ci_setup(ci_arm) + sh ( + script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_frontend_cpu.sh", + label: 'Run Python frontend tests', + ) + }) } + } finally { + junit 'build/pytest-results/*.xml' } } - } else { - Utils.markStageSkippedForConditional('frontend: GPU 1 of 3') } + } else { + Utils.markStageSkippedForConditional('frontend: aarch64 2 of 2') + } +} + + + +def test() { +stage('Test') { + environment { + SKIP_SLOW_TESTS = "${skip_slow_tests}" + } + parallel( + 'unittest: GPU 1 of 3': { + shard_run_unittest_GPU_1_of_3() + }, + 'unittest: GPU 2 of 3': { + shard_run_unittest_GPU_2_of_3() + }, + 'unittest: GPU 3 of 3': { + shard_run_unittest_GPU_3_of_3() + }, + 'integration: CPU 1 of 6': { + shard_run_integration_CPU_1_of_6() }, - 'frontend: GPU 2 of 3': { + 'integration: CPU 2 of 6': { + shard_run_integration_CPU_2_of_6() + }, + 'integration: CPU 3 of 6': { + shard_run_integration_CPU_3_of_6() + }, + 'integration: CPU 4 of 6': { + shard_run_integration_CPU_4_of_6() + }, + 'integration: CPU 5 of 6': { + shard_run_integration_CPU_5_of_6() + }, + 'integration: CPU 6 of 6': { + shard_run_integration_CPU_6_of_6() + }, + 'python: i386 1 of 5': { + shard_run_python_i386_1_of_5() + }, + 'python: i386 2 of 5': { + shard_run_python_i386_2_of_5() + }, + 'python: i386 3 of 5': { + shard_run_python_i386_3_of_5() + }, + 'python: i386 4 of 5': { + shard_run_python_i386_4_of_5() + }, + 'python: i386 5 of 5': { + shard_run_python_i386_5_of_5() + }, + 'test: Hexagon 1 of 7': { + shard_run_test_Hexagon_1_of_7() + }, + 'test: Hexagon 2 of 7': { + shard_run_test_Hexagon_2_of_7() + }, + 'test: Hexagon 3 of 7': { + shard_run_test_Hexagon_3_of_7() + }, + 'test: Hexagon 4 of 7': { + shard_run_test_Hexagon_4_of_7() + }, + 'test: Hexagon 5 of 7': { + shard_run_test_Hexagon_5_of_7() + }, + 'test: Hexagon 6 of 7': { + shard_run_test_Hexagon_6_of_7() + }, + 'test: Hexagon 7 of 7': { + shard_run_test_Hexagon_7_of_7() + }, + 'integration: aarch64 1 of 4': { + shard_run_integration_aarch64_1_of_4() + }, + 'integration: aarch64 2 of 4': { + shard_run_integration_aarch64_2_of_4() + }, + 'integration: aarch64 3 of 4': { + shard_run_integration_aarch64_3_of_4() + }, + 'integration: aarch64 4 of 4': { + shard_run_integration_aarch64_4_of_4() + }, + 'topi: GPU 1 of 4': { + shard_run_topi_GPU_1_of_4() + }, + 'topi: GPU 2 of 4': { + shard_run_topi_GPU_2_of_4() + }, + 'topi: GPU 3 of 4': { + shard_run_topi_GPU_3_of_4() + }, + 'topi: GPU 4 of 4': { + shard_run_topi_GPU_4_of_4() + }, + 'frontend: GPU 1 of 6': { + shard_run_frontend_GPU_1_of_6() + }, + 'frontend: GPU 2 of 6': { + shard_run_frontend_GPU_2_of_6() + }, + 'frontend: GPU 3 of 6': { + shard_run_frontend_GPU_3_of_6() + }, + 'frontend: GPU 4 of 6': { + shard_run_frontend_GPU_4_of_6() + }, + 'frontend: GPU 5 of 6': { + shard_run_frontend_GPU_5_of_6() + }, + 'frontend: GPU 6 of 6': { + shard_run_frontend_GPU_6_of_6() + }, + 'topi: aarch64 1 of 2': { + shard_run_topi_aarch64_1_of_2() + }, + 'topi: aarch64 2 of 2': { + shard_run_topi_aarch64_2_of_2() + }, + 'frontend: aarch64 1 of 2': { + shard_run_frontend_aarch64_1_of_2() + }, + 'frontend: aarch64 2 of 2': { + shard_run_frontend_aarch64_2_of_2() + }, + 'unittest: CPU': { if (!skip_ci && is_docs_only_build != 1) { - node('GPU') { - ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") { - try { - init_git() - timeout(time: max_time, unit: 'MINUTES') { - withEnv([ - 'PLATFORM=gpu', - 'TVM_NUM_SHARDS=3', - 'TVM_SHARD_INDEX=1'], { + node('CPU-SMALL') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-cpu") { + timeout(time: max_time, unit: 'MINUTES') { + try { + init_git() + withEnv(['PLATFORM=cpu'], { sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_tsim.so build/libvta_tsim.so + md5sum build/libvta_tsim.so + aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so + aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libvta_fsim.so build/libvta_fsim.so md5sum build/libvta_fsim.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake + aws s3 cp --no-progress s3://${s3_prefix}/cpu/build/config.cmake build/config.cmake md5sum build/config.cmake """, label: 'Download artifacts from S3', ) - ci_setup(ci_gpu) + ci_setup(ci_cpu) + cpp_unittest(ci_cpu) + python_unittest(ci_cpu) + fsim_test(ci_cpu) sh ( - script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_frontend.sh", - label: 'Run Python frontend tests', + script: "${docker_run} ${ci_cpu} ./tests/scripts/task_python_vta_tsim.sh", + label: 'Run VTA tests in TSIM', ) }) + } finally { + junit 'build/pytest-results/*.xml' } - } finally { - junit 'build/pytest-results/*.xml' } } } } else { - Utils.markStageSkippedForConditional('frontend: GPU 2 of 3') + Utils.markStageSkippedForConditional('unittest: CPU') } }, - 'frontend: GPU 3 of 3': { + 'test: QEMU': { if (!skip_ci && is_docs_only_build != 1) { - node('GPU') { - ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") { - try { - init_git() - timeout(time: max_time, unit: 'MINUTES') { - withEnv([ - 'PLATFORM=gpu', - 'TVM_NUM_SHARDS=3', - 'TVM_SHARD_INDEX=2'], { + node('CPU-SMALL') { + ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-qemu") { + timeout(time: max_time, unit: 'MINUTES') { + try { + init_git() + withEnv(['PLATFORM=qemu'], { sh( script: """ set -eux - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm.so build/libtvm.so + aws s3 cp --no-progress s3://${s3_prefix}/qemu/build/libtvm.so build/libtvm.so md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libvta_fsim.so build/libvta_fsim.so - md5sum build/libvta_fsim.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/libtvm_runtime.so build/libtvm_runtime.so + aws s3 cp --no-progress s3://${s3_prefix}/qemu/build/libtvm_runtime.so build/libtvm_runtime.so md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/gpu/build/config.cmake build/config.cmake + aws s3 cp --no-progress s3://${s3_prefix}/qemu/build/config.cmake build/config.cmake md5sum build/config.cmake + aws s3 cp --no-progress s3://${s3_prefix}/qemu/build/microtvm_template_projects build/microtvm_template_projects --recursive """, label: 'Download artifacts from S3', ) - ci_setup(ci_gpu) + add_microtvm_permissions() + ci_setup(ci_qemu) + cpp_unittest(ci_qemu) sh ( - script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_frontend.sh", - label: 'Run Python frontend tests', + script: "${docker_run} ${ci_qemu} ./tests/scripts/task_python_microtvm.sh", + label: 'Run microTVM tests', + ) + sh ( + script: "${docker_run} ${ci_qemu} ./tests/scripts/task_demo_microtvm.sh", + label: 'Run microTVM demos', ) }) + } finally { + junit 'build/pytest-results/*.xml' } - } finally { - junit 'build/pytest-results/*.xml' } } } } else { - Utils.markStageSkippedForConditional('frontend: GPU 3 of 3') + Utils.markStageSkippedForConditional('test: QEMU') } }, 'frontend: CPU': { if (!skip_ci && is_docs_only_build != 1) { - node('CPU') { + node('CPU-SMALL') { ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-cpu") { timeout(time: max_time, unit: 'MINUTES') { try { @@ -1710,45 +2752,6 @@ stage('Test') { Utils.markStageSkippedForConditional('frontend: CPU') } }, - 'frontend: aarch64': { - if (!skip_ci && is_docs_only_build != 1) { - node('ARM') { - ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-arm") { - timeout(time: max_time, unit: 'MINUTES') { - try { - init_git() - withEnv(['PLATFORM=arm'], { - sh( - script: """ - set -eux - aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm.so build/libtvm.so - md5sum build/libtvm.so - aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libvta_fsim.so build/libvta_fsim.so - md5sum build/libvta_fsim.so - aws s3 cp --no-progress s3://${s3_prefix}/arm/build/libtvm_runtime.so build/libtvm_runtime.so - md5sum build/libtvm_runtime.so - aws s3 cp --no-progress s3://${s3_prefix}/arm/build/config.cmake build/config.cmake - md5sum build/config.cmake - """, - label: 'Download artifacts from S3', - ) - - ci_setup(ci_arm) - sh ( - script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_frontend_cpu.sh", - label: 'Run Python frontend tests', - ) - }) - } finally { - junit 'build/pytest-results/*.xml' - } - } - } - } - } else { - Utils.markStageSkippedForConditional('frontend: aarch64') - } - }, 'docs: GPU': { if (!skip_ci) { node('GPU') { diff --git a/apps/android_camera/README.md b/apps/android_camera/README.md index c292ce40c582..f659e905f281 100644 --- a/apps/android_camera/README.md +++ b/apps/android_camera/README.md @@ -1,28 +1,107 @@ -[//]: # Licensed to the Apache Software Foundation (ASF) under one -[//]: # or more contributor license agreements. See the NOTICE file -[//]: # distributed with this work for additional information -[//]: # regarding copyright ownership. The ASF licenses this file -[//]: # to you under the Apache License, Version 2.0 (the -[//]: # "License"); you may not use this file except in compliance -[//]: # with the License. You may obtain a copy of the License at -[//]: # -[//]: # http://www.apache.org/licenses/LICENSE-2.0 -[//]: # -[//]: # Unless required by applicable law or agreed to in writing, -[//]: # software distributed under the License is distributed on an -[//]: # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -[//]: # KIND, either express or implied. See the License for the -[//]: # specific language governing permissions and limitations -[//]: # under the License. - -Android Camera Demo Sample App -============================== - -The Android Camera Demo Sample App provides a basic implementation of an Android -app that uses the tvm runtime to perform image classification in real time. - -Converting Models ------------------ + + + + + + + + + + + + + + + + + + +# Android Camera Demo Sample App + +The Android Camera Demo Sample App provides a basic implementation of an Android app that uses the tvm runtime to perform image classification in real time. + +You will need JDK, [Android NDK](https://developer.android.com/ndk) and an Android device to use this. + +## Build and Installation + +### Prepare Models The `models/prepare_models.py` script provides a example flow for dumping model parameter files for use by the app. + +1. Set path to the NDK CC: `export TVM_NDK_CC=[Path to CC, e.g. /opt/android-toolchain-arm64/bin/aarch64-linux-android-g++]` +2. Switch to the script directory: `cd models` +3. Run script: `python3 prepare_model.py` + +#### Sample output +``` +mobilenet_v2 +getting model... +building... +dumping lib... +dumping graph... +dumping params... +dumping labels... +resnet18_v1 +getting model... +building... +dumping lib... +dumping graph... +dumping params... +dumping labels... +``` + +### Build APK + +We use [Gradle](https://gradle.org) to build. Please follow [the installation instruction](https://gradle.org/install) for your operating system. + +Before you build the Android application, please refer to [TVM4J Installation Guide](https://github.com/apache/tvm/blob/main/jvm/README.md) and install tvm4j-core to your local maven repository. You can find tvm4j dependency declare in `app/build.gradle`. Modify it if it is necessary. + +``` +dependencies { + implementation fileTree(dir: 'libs', include: ['*.jar']) + androidTestImplementation('androidx.test.espresso:espresso-core:3.2.0', { + exclude group: 'com.android.support', module: 'support-annotations' + }) + implementation 'androidx.appcompat:appcompat:1.4.0' + implementation 'androidx.constraintlayout:constraintlayout:2.1.3' + implementation 'com.google.android.material:material:1.5.0' + implementation 'org.apache.tvm:tvm4j-core:0.0.1-SNAPSHOT' + testImplementation 'junit:junit:4.13.2' + + implementation "androidx.concurrent:concurrent-futures:1.0.0" + implementation "androidx.camera:camera-core:1.0.0-beta01" + implementation "androidx.camera:camera-camera2:1.0.0-beta01" + implementation "androidx.camera:camera-view:1.0.0-alpha08" + implementation "androidx.camera:camera-extensions:1.0.0-alpha08" + implementation "androidx.camera:camera-lifecycle:1.0.0-beta01" +} +``` + +Now use Gradle to compile JNI, resolve Java dependencies and build the Android application together with tvm4j. Run following script to generate the apk file. + +```bash +export ANDROID_HOME=[Path to your Android SDK, e.g., ~/Android/sdk] +cd apps/android_camera +gradle clean build +``` + +In `app/build/outputs/apk` you'll find `app-release-unsigned.apk`, use `dev_tools/gen_keystore.sh` to generate a signature and use `dev_tools/sign_apk.sh` to get the signed apk file `app/build/outputs/apk/release/tv8mdemo-release.apk`. + +Upload `tv8mdemo-release.apk` to your Android device and install it: + +```bash +$ANDROID_HOME/platform-tools/adb install app/build/outputs/apk/release/tv8mdemo-release.apk +``` + +If you see error: + + adb: failed to install app/build/outputs/apk/release/tv8mdemo-release.apk: + Failure [INSTALL_FAILED_UPDATE_INCOMPATIBLE: + Package ml.apache.tvm.android.androidcamerademo signatures do not match the previously installed version; ignoring!] + +Run uninstall first: + +```bash +$ANDROID_HOME/platform-tools/adb uninstall ml.apache.tvm.android.androidcamerademo +``` diff --git a/apps/android_camera/app/build.gradle b/apps/android_camera/app/build.gradle index 8a772a3d29f3..c7767559e4df 100644 --- a/apps/android_camera/app/build.gradle +++ b/apps/android_camera/app/build.gradle @@ -17,20 +17,47 @@ apply plugin: 'com.android.application' +task generateJniHeaders(type: Exec, description: 'Generate JNI Headers') { + def headerPath = "${project.projectDir}/src/main/jni" + def classPath = "${project.projectDir}/../../../jvm/core/target/*" + def filePath = "${project.projectDir}/../../../jvm/core/src/main/java/org/apache/tvm/LibInfo.java" + commandLine "javac", "-h", headerPath, "-classpath", classPath, filePath + doLast { + file("${headerPath}/org_apache_tvm_LibInfo.h").renameTo(file("${headerPath}/org_apache_tvm_native_c_api.h")) + } +} + +task copyFiles(type: Copy, description: 'Copy Sources for ndk-build') { + dependsOn "generateJniHeaders" + def ndkFilesPath = "${project.projectDir}/../../../jvm/native/src/main/native" + def srcPath = "${project.projectDir}/src/main/jni/" + + from "${ndkFilesPath}/org_apache_tvm_native_c_api.cc", "${ndkFilesPath}/jni_helper_func.h" + into srcPath +} + +task deleteLibs(type: Delete, description: "Delete Compiled Libraries") { + dependsOn "copyFiles" + def libsPath = "${project.projectDir}/src/main/libs" + delete libsPath +} + task buildJni(type: Exec, description: 'Build JNI libs') { - commandLine 'sh', 'src/main/jni/build.sh' + dependsOn "deleteLibs" + def buildPath = "${project.projectDir}/src/main/jni" + commandLine "ndk-build", "--directory", buildPath } tasks.withType(JavaCompile) { - //compileTask -> compileTask.dependsOn buildJni + compileTask -> compileTask.dependsOn buildJni } android { - compileSdkVersion 29 + compileSdkVersion 31 defaultConfig { applicationId "ml.apache.tvm.android.androidcamerademo" minSdkVersion 24 - targetSdkVersion 29 + targetSdkVersion 26 renderscriptTargetApi 18 renderscriptSupportModeEnabled true versionCode 1 @@ -53,7 +80,10 @@ android { sourceCompatibility JavaVersion.VERSION_1_8 targetCompatibility JavaVersion.VERSION_1_8 } - buildToolsVersion = '29.0.3' + + lintOptions { + disable "Instantiatable" // MainActivity and RPCActivity must extend android.app.Activity + } } dependencies { @@ -61,13 +91,13 @@ dependencies { androidTestImplementation('androidx.test.espresso:espresso-core:3.2.0', { exclude group: 'com.android.support', module: 'support-annotations' }) - implementation 'androidx.appcompat:appcompat:1.1.0' - implementation 'androidx.constraintlayout:constraintlayout:1.1.3' - implementation 'com.google.android.material:material:1.1.0' - implementation 'org.apache.tvm:tvm4j-core:0.0.1-SNAPSHOT' - testImplementation 'junit:junit:4.13' + implementation 'androidx.appcompat:appcompat:1.4.0' + implementation 'androidx.constraintlayout:constraintlayout:2.1.3' + implementation 'com.google.android.material:material:1.5.0' + implementation files('../../../jvm/core/target/tvm4j-core-0.0.1-SNAPSHOT.jar') + testImplementation 'junit:junit:4.13.2' - implementation("androidx.concurrent:concurrent-futures:1.0.0") + implementation "androidx.concurrent:concurrent-futures:1.0.0" implementation "androidx.camera:camera-core:1.0.0-beta01" implementation "androidx.camera:camera-camera2:1.0.0-beta01" // If you want to use the CameraX View class diff --git a/apps/android_camera/app/src/main/AndroidManifest.xml b/apps/android_camera/app/src/main/AndroidManifest.xml index 0821286d5543..e5b6465c5874 100644 --- a/apps/android_camera/app/src/main/AndroidManifest.xml +++ b/apps/android_camera/app/src/main/AndroidManifest.xml @@ -28,7 +28,7 @@ tools:ignore="AllowBackup,MissingApplicationIcon"> diff --git a/apps/android_camera/app/src/main/java/org/apache/tvm/android/androidcamerademo/Camera2BasicFragment.java b/apps/android_camera/app/src/main/java/org/apache/tvm/android/androidcamerademo/Camera2BasicFragment.java index 8a5f54a3e399..3a55a62d739c 100644 --- a/apps/android_camera/app/src/main/java/org/apache/tvm/android/androidcamerademo/Camera2BasicFragment.java +++ b/apps/android_camera/app/src/main/java/org/apache/tvm/android/androidcamerademo/Camera2BasicFragment.java @@ -382,7 +382,7 @@ private Bitmap YUV_420_888_toRGB(Image image, int width, int height) { } private float[] getFrame(ImageProxy imageProxy) { - @SuppressLint("UnsafeExperimentalUsageError") + @SuppressLint("UnsafeOptInUsageError") Image image = imageProxy.getImage(); // extract the jpeg content if (image == null) { diff --git a/apps/android_camera/app/src/main/java/org/apache/tvm/android/androidcamerademo/MainActivity.java b/apps/android_camera/app/src/main/java/org/apache/tvm/android/androidcamerademo/MainActivity.java index f9c573a5d1fe..06b1c9730d05 100644 --- a/apps/android_camera/app/src/main/java/org/apache/tvm/android/androidcamerademo/MainActivity.java +++ b/apps/android_camera/app/src/main/java/org/apache/tvm/android/androidcamerademo/MainActivity.java @@ -78,6 +78,7 @@ private void startFragment() { @Override public void onRequestPermissionsResult( int requestCode, @NonNull String[] permissions, @NonNull int[] grantResults) { + super.onRequestPermissionsResult(requestCode, permissions, grantResults); if (allPermissionsGranted()) { startFragment(); } else { diff --git a/apps/android_camera/app/src/main/jni/Android.mk b/apps/android_camera/app/src/main/jni/Android.mk index a5eacb0c0c2d..4ff3da8f3327 100644 --- a/apps/android_camera/app/src/main/jni/Android.mk +++ b/apps/android_camera/app/src/main/jni/Android.mk @@ -34,11 +34,14 @@ endif include $(config) LOCAL_SRC_FILES := org_apache_tvm_native_c_api.cc + LOCAL_LDFLAGS := -L$(SYSROOT)/usr/lib/ -llog LOCAL_C_INCLUDES := $(ROOT_PATH)/include \ + $(ROOT_PATH)/src/runtime/rpc \ $(ROOT_PATH)/3rdparty/dlpack/include \ $(ROOT_PATH)/3rdparty/dmlc-core/include \ + $(MY_PATH) LOCAL_MODULE = tvm4j_runtime_packed diff --git a/apps/android_camera/app/src/main/jni/tvm_runtime.h b/apps/android_camera/app/src/main/jni/tvm_runtime.h index bed3bc82d5a3..b20227b34db4 100644 --- a/apps/android_camera/app/src/main/jni/tvm_runtime.h +++ b/apps/android_camera/app/src/main/jni/tvm_runtime.h @@ -40,11 +40,16 @@ #include "../src/runtime/graph_executor/graph_executor.cc" #include "../src/runtime/library_module.cc" #include "../src/runtime/logging.cc" +#include "../src/runtime/minrpc/minrpc_logger.cc" #include "../src/runtime/module.cc" #include "../src/runtime/ndarray.cc" #include "../src/runtime/object.cc" +#include "../src/runtime/profiling.cc" #include "../src/runtime/registry.cc" +#include "../src/runtime/rpc/rpc_channel.cc" +#include "../src/runtime/rpc/rpc_endpoint.cc" #include "../src/runtime/rpc/rpc_event_impl.cc" +#include "../src/runtime/rpc/rpc_local_session.cc" #include "../src/runtime/rpc/rpc_module.cc" #include "../src/runtime/rpc/rpc_server_env.cc" #include "../src/runtime/rpc/rpc_session.cc" diff --git a/apps/android_camera/app/src/main/res/layout/listview_row.xml b/apps/android_camera/app/src/main/res/layout/listview_row.xml index 4c233dc40379..5038a27557e9 100644 --- a/apps/android_camera/app/src/main/res/layout/listview_row.xml +++ b/apps/android_camera/app/src/main/res/layout/listview_row.xml @@ -20,7 +20,7 @@ android:id="@+id/listview_row_text" android:layout_width="match_parent" android:layout_height="match_parent" - android:layout_marginRight="2dp" + android:layout_marginEnd="2dp" android:background="@drawable/item_selector" android:padding="10dp" android:textSize="18sp" diff --git a/apps/android_camera/build.gradle b/apps/android_camera/build.gradle index a58bc631dac5..1cd5ac9b656f 100644 --- a/apps/android_camera/build.gradle +++ b/apps/android_camera/build.gradle @@ -19,14 +19,14 @@ buildscript { repositories { - jcenter() + gradlePluginPortal() maven { url 'https://maven.google.com' } - google() } dependencies { - classpath 'com.android.tools.build:gradle:3.6.1' + classpath 'com.android.tools.build:gradle:7.1.2' + // NOTE: Do not place your application dependencies here; they belong // in the individual module build.gradle files } @@ -34,16 +34,15 @@ buildscript { allprojects { repositories { - jcenter() + gradlePluginPortal() maven { - url 'https://maven.google.com' + url 'https://maven.google.com' } mavenLocal() mavenCentral() - google() } } task clean(type: Delete) { delete rootProject.buildDir -} \ No newline at end of file +} diff --git a/apps/android_camera/gradle.properties b/apps/android_camera/gradle.properties index f1328bf5fd43..1add1b540993 100644 --- a/apps/android_camera/gradle.properties +++ b/apps/android_camera/gradle.properties @@ -1,4 +1,3 @@ - # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information diff --git a/apps/android_camera/models/prepare_model.py b/apps/android_camera/models/prepare_model.py index 2ea1b0a120af..959e93f8b47b 100644 --- a/apps/android_camera/models/prepare_model.py +++ b/apps/android_camera/models/prepare_model.py @@ -106,7 +106,7 @@ def main(model_str, output_path): f.write(graph) print("dumping params...") with open(output_path_str + "/" + "deploy_param.params", "wb") as f: - f.write(runtime.save_param_dict(params)) + f.write(tvm.runtime.save_param_dict(params)) print("dumping labels...") synset_url = "".join( [ diff --git a/apps/android_camera/models/requirements.txt b/apps/android_camera/models/requirements.txt new file mode 100644 index 000000000000..98aa53def46f --- /dev/null +++ b/apps/android_camera/models/requirements.txt @@ -0,0 +1,4 @@ +keras +mxnet +scipy +tensorflow \ No newline at end of file diff --git a/apps/android_deploy/app/build.gradle b/apps/android_deploy/app/build.gradle index c00528ba49fd..2949775349bb 100644 --- a/apps/android_deploy/app/build.gradle +++ b/apps/android_deploy/app/build.gradle @@ -26,8 +26,35 @@ apply from: "download-models.gradle" apply plugin: 'com.android.application' +task generateJniHeaders(type: Exec, description: 'Generate JNI Headers') { + def headerPath = "${project.projectDir}/src/main/jni" + def classPath = "${project.projectDir}/../../../jvm/core/target/*" + def filePath = "${project.projectDir}/../../../jvm/core/src/main/java/org/apache/tvm/LibInfo.java" + commandLine "javac", "-h", headerPath, "-classpath", classPath, filePath + doLast { + file("${headerPath}/org_apache_tvm_LibInfo.h").renameTo(file("${headerPath}/org_apache_tvm_native_c_api.h")) + } +} + +task copyFiles(type: Copy, description: 'Copy Sources for ndk-build') { + dependsOn "generateJniHeaders" + def ndkFilesPath = "${project.projectDir}/../../../jvm/native/src/main/native" + def srcPath = "${project.projectDir}/src/main/jni/" + + from "${ndkFilesPath}/org_apache_tvm_native_c_api.cc", "${ndkFilesPath}/jni_helper_func.h" + into srcPath +} + +task deleteLibs(type: Delete, description: "Delete Compiled Libraries") { + dependsOn "copyFiles" + def libsPath = "${project.projectDir}/src/main/libs" + delete libsPath +} + task buildJni(type: Exec, description: 'Build JNI libs') { - commandLine 'sh', 'src/main/jni/build.sh' + dependsOn "deleteLibs" + def buildPath = "${project.projectDir}/src/main/jni" + commandLine "ndk-build", "--directory", buildPath } tasks.withType(JavaCompile) { @@ -35,11 +62,10 @@ tasks.withType(JavaCompile) { } android { - compileSdkVersion 26 - buildToolsVersion "26.0.1" + compileSdkVersion 31 defaultConfig { applicationId "org.apache.tvm.android.demo" - minSdkVersion 17 + minSdkVersion 24 targetSdkVersion 26 versionCode 1 versionName "1.0" @@ -55,19 +81,24 @@ android { main { jni.srcDirs = [] jniLibs.srcDirs = ['src/main/libs'] - assets.srcDirs = [project.ext.ASSET_DIR] } } + + lintOptions { + disable "Instantiatable" // MainActivity and RPCActivity must extend android.app.Activity + disable "MissingApplicationIcon" // Should explicitly set android:icon, there is no default + disable "UnsafeNativeCodeLocation" // Shared libraries should not be placed in the res or assets directories. + } } dependencies { - compile fileTree(dir: 'libs', include: ['*.jar']) - androidTestCompile('com.android.support.test.espresso:espresso-core:2.2.2', { + implementation fileTree(dir: 'libs', include: ['*.jar']) + androidTestImplementation('com.android.support.test.espresso:espresso-core:3.4.0', { exclude group: 'com.android.support', module: 'support-annotations' }) - compile 'com.android.support:appcompat-v7:26.0.1' - compile 'com.android.support.constraint:constraint-layout:1.0.2' - compile 'com.android.support:design:26.0.1' - compile 'org.apache.tvm:tvm4j-core:0.0.1-SNAPSHOT' - testCompile 'junit:junit:4.12' -} + implementation 'androidx.appcompat:appcompat:1.4.1' + implementation 'com.android.support.constraint:constraint-layout:2.1.3' + implementation 'com.android.support:design:28.0.0' + implementation files('../../../jvm/core/target/tvm4j-core-0.0.1-SNAPSHOT.jar') + testImplementation 'junit:junit:4.13.2' +} \ No newline at end of file diff --git a/apps/android_deploy/app/download-models.gradle b/apps/android_deploy/app/download-models.gradle index ed660e0221ee..4d1620bfd953 100644 --- a/apps/android_deploy/app/download-models.gradle +++ b/apps/android_deploy/app/download-models.gradle @@ -34,7 +34,7 @@ buildscript { jcenter() } dependencies { - classpath 'de.undercouch:gradle-download-task:3.2.0' + classpath 'de.undercouch:gradle-download-task:5.0.4' } } diff --git a/apps/android_deploy/app/src/main/AndroidManifest.xml b/apps/android_deploy/app/src/main/AndroidManifest.xml index bf3463f4d9c6..ce1b1fac4552 100644 --- a/apps/android_deploy/app/src/main/AndroidManifest.xml +++ b/apps/android_deploy/app/src/main/AndroidManifest.xml @@ -25,6 +25,7 @@ under the License. + + android:exported="true" + android:screenOrientation="unspecified"> @@ -52,6 +53,4 @@ under the License. - - diff --git a/apps/android_deploy/app/src/main/java/org/apache/tvm/android/demo/MainActivity.java b/apps/android_deploy/app/src/main/java/org/apache/tvm/android/demo/MainActivity.java index 85cc7a277b4d..6320b6aa8afd 100644 --- a/apps/android_deploy/app/src/main/java/org/apache/tvm/android/demo/MainActivity.java +++ b/apps/android_deploy/app/src/main/java/org/apache/tvm/android/demo/MainActivity.java @@ -35,9 +35,9 @@ import android.os.Environment; import android.os.SystemClock; import android.provider.MediaStore; -import android.support.v4.content.FileProvider; -import android.support.v7.app.AppCompatActivity; -import android.support.v7.widget.Toolbar; +import androidx.core.content.FileProvider; +import androidx.appcompat.app.AppCompatActivity; +import androidx.appcompat.widget.Toolbar; import android.util.Log; import android.view.View; import android.widget.ImageView; @@ -51,6 +51,7 @@ import java.io.IOException; import java.text.SimpleDateFormat; import java.util.Date; +import java.util.Locale; import java.util.Vector; import org.apache.tvm.Function; @@ -487,7 +488,7 @@ private final String getTempLibFilePath(String fileName) throws IOException { */ private File createImageFile() { // Create an image file name - String timeStamp = new SimpleDateFormat("yyyyMMdd_HHmmss").format(new Date()); + String timeStamp = new SimpleDateFormat("yyyyMMdd_HHmmss", Locale.US).format(new Date()); String imageFileName = "JPEG_" + timeStamp + "_"; File storageDir = Environment.getExternalStoragePublicDirectory( Environment.DIRECTORY_PICTURES); @@ -527,6 +528,7 @@ public void onClick(DialogInterface dialog, int id) { @Override public void onRequestPermissionsResult (final int requestCode, final String[] permissions, final int[] grantResults){ + super.onRequestPermissionsResult(requestCode, permissions, grantResults); if (requestCode == PERMISSIONS_REQUEST) { if (grantResults.length > 0 && grantResults[0] == PackageManager.PERMISSION_GRANTED diff --git a/apps/android_deploy/app/src/main/res/layout/activity_main.xml b/apps/android_deploy/app/src/main/res/layout/activity_main.xml index 0778374223d8..4b019e1fbdb3 100644 --- a/apps/android_deploy/app/src/main/res/layout/activity_main.xml +++ b/apps/android_deploy/app/src/main/res/layout/activity_main.xml @@ -19,29 +19,28 @@ specific language governing permissions and limitations under the License. --> - - - + android:layout_height="match_parent" + tools:context="org.apache.tvm.tvmrpc.MainActivity"> - + android:theme="@style/AppTheme.AppBarOverlay"> - + - + - + + diff --git a/apps/android_deploy/app/src/main/res/layout/content_main.xml b/apps/android_deploy/app/src/main/res/layout/content_main.xml index aa1b0ea72b3e..6bf3c19f7f81 100644 --- a/apps/android_deploy/app/src/main/res/layout/content_main.xml +++ b/apps/android_deploy/app/src/main/res/layout/content_main.xml @@ -35,10 +35,10 @@ under the License.