From 1cdc156ae9963461e47777c0e73eb29029d26267 Mon Sep 17 00:00:00 2001
From: Wei Chu <weichu@amazon.com>
Date: Thu, 1 Apr 2021 22:03:07 -0700
Subject: [PATCH] split cv tests

---
 ci/docker/runtime_functions.sh                  | 13 +++++++++++--
 pytest.ini                                      |  8 +++++---
 tests/nightly/JenkinsfileForBinaries            | 12 ++++++++++--
 tests/python-pytest/onnx/test_onnxruntime_cv.py |  6 ++++++
 4 files changed, 32 insertions(+), 7 deletions(-)

diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 7f11151bdca1..2c3f36769f37 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -1594,13 +1594,22 @@ nightly_estimator() {
     nosetests test_sentiment_rnn.py
 }
 
-nightly_onnx_cv_tests() {
+nightly_onnx_cv_batch1_tests() {
     set -ex
     export PYTHONPATH=./python/
     export MXNET_SUBGRAPH_VERBOSE=0
     export DMLC_LOG_STACK_TRACE_DEPTH=10
     COV_ARG="--cov=./ --cov-report=xml --cov-append"
-    pytest $COV_ARG --verbose tests/python-pytest/onnx/test_onnxruntime_cv.py
+    pytest $COV_ARG -v -m onnx_cv_batch1 tests/python-pytest/onnx/test_onnxruntime_cv.py
+}
+
+nightly_onnx_cv_batch2_tests() {
+    set -ex
+    export PYTHONPATH=./python/
+    export MXNET_SUBGRAPH_VERBOSE=0
+    export DMLC_LOG_STACK_TRACE_DEPTH=10
+    COV_ARG="--cov=./ --cov-report=xml --cov-append"
+    pytest $COV_ARG -v -m onnx_cv_batch2 tests/python-pytest/onnx/test_onnxruntime_cv.py
 }
 
 nightly_onnx_nlp_tests() {
diff --git a/pytest.ini b/pytest.ini
index 55783a14ffa6..8286f759590e 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -17,13 +17,15 @@
 
 [pytest]
 markers =
-    seed: set the python, numpy and mxnet random seeds to a specified value for test reproducibility
+    seed: set the python, numpy and mxnet random seeds to a specified value for test reproducibility.
     serial: mark a test that requires more resources to run that are thus only suitable for serial run.
     remote_required: mark a test that requires internet access.
     gpu: mark a test that requires GPU.
-    integration: mark an integration test
-    onnx_coverage: ONNX coverage test
+    integration: mark an integration test.
+    onnx_coverage: ONNX coverage test.
     garbage_expected: this test leaks ndarray references. The tested functionality is broken or there is a Python bug.
+    onnx_cv_batch1: mark a test that runs in batch1 of nightly ONNX CV tests.
+    onnx_cv_batch2: mark a test that runs in batch2 of nightly ONNX CV tests.
 
 env =
     MXNET_HOME=tests/data
diff --git a/tests/nightly/JenkinsfileForBinaries b/tests/nightly/JenkinsfileForBinaries
index 1c1027ad6be5..939cb0705f23 100755
--- a/tests/nightly/JenkinsfileForBinaries
+++ b/tests/nightly/JenkinsfileForBinaries
@@ -94,11 +94,19 @@ core_logic: {
         }
       }
     },
-    'ONNX-CV: CPU': {
+    'ONNX-CV-batch1: CPU': {
       node(NODE_LINUX_CPU) {
         ws('workspace/onnx-cv-test-cpu') {
           utils.unpack_and_init('cpu_int64', mx_cmake_lib)
-          utils.docker_run('ubuntu_nightly_cpu', 'nightly_onnx_cv_tests', false)
+          utils.docker_run('ubuntu_nightly_cpu', 'nightly_onnx_cv_batch1_tests', false)
+        }
+      }
+    },
+    'ONNX-CV-batch2: CPU': {
+      node(NODE_LINUX_CPU) {
+        ws('workspace/onnx-cv-test-cpu') {
+          utils.unpack_and_init('cpu_int64', mx_cmake_lib)
+          utils.docker_run('ubuntu_nightly_cpu', 'nightly_onnx_cv_batch2_tests', false)
         }
       }
     },
diff --git a/tests/python-pytest/onnx/test_onnxruntime_cv.py b/tests/python-pytest/onnx/test_onnxruntime_cv.py
index 4e455147dcdd..fd6284a9434b 100644
--- a/tests/python-pytest/onnx/test_onnxruntime_cv.py
+++ b/tests/python-pytest/onnx/test_onnxruntime_cv.py
@@ -88,6 +88,7 @@ def obj_class_test_images(tmpdir_factory):
         paths.append(fn)
     return paths
 
+@pytest.mark.onnx_cv_batch1
 @pytest.mark.parametrize('model', [
     'alexnet',
     'cifar_resnet20_v1',
@@ -215,6 +216,7 @@ def obj_detection_test_images(tmpdir_factory):
     return paths
 
 
+@pytest.mark.onnx_cv_batch2
 @pytest.mark.parametrize('model', [
     'center_net_resnet18_v1b_voc',
     'center_net_resnet50_v1b_voc',
@@ -335,6 +337,7 @@ def img_segmentation_test_images(tmpdir_factory):
         paths.append(fn)
     return paths
 
+@pytest.mark.onnx_cv_batch2
 @pytest.mark.parametrize('model', [
     'fcn_resnet50_ade',
     'fcn_resnet101_ade',
@@ -401,6 +404,7 @@ def pose_estimation_test_images(tmpdir_factory):
         paths.append(fn)
     return paths
 
+@pytest.mark.onnx_cv_batch1
 @pytest.mark.parametrize('model', [
     'simple_pose_resnet18_v1b',
     'simple_pose_resnet50_v1b',
@@ -462,6 +466,7 @@ def act_recognition_test_data(tmpdir_factory):
         paths.append(fn)
     return paths
 
+@pytest.mark.onnx_cv_batch2
 @pytest.mark.parametrize('model', [
     'inceptionv1_kinetics400',
     'resnet18_v1b_kinetics400',
@@ -507,6 +512,7 @@ def load_video(filepath):
 
 
 @with_seed()
+@pytest.mark.onnx_cv_batch1
 @pytest.mark.integration
 @pytest.mark.parametrize('model_name', ['mobilenet1.0', 'inceptionv3', 'darknet53', 'resnest14'])
 def test_dynamic_shape_cv_inference_onnxruntime(tmp_path, model_name):