diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs index 15986bbb603a..fd8ddc45da14 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs @@ -63,8 +63,7 @@ public void TestSessionOptions() opt.GraphOptimizationLevel = GraphOptimizationLevel.ORT_ENABLE_EXTENDED; Assert.Equal(GraphOptimizationLevel.ORT_ENABLE_EXTENDED, opt.GraphOptimizationLevel); - - Assert.Throws(() => { opt.ThreadPoolSize = -2; }); + Assert.Throws(() => { opt.GraphOptimizationLevel = (GraphOptimizationLevel)10; }); } } diff --git a/include/onnxruntime/core/session/onnxruntime_c_api.h b/include/onnxruntime/core/session/onnxruntime_c_api.h index 8566125ac14d..6025a8ac8066 100644 --- a/include/onnxruntime/core/session/onnxruntime_c_api.h +++ b/include/onnxruntime/core/session/onnxruntime_c_api.h @@ -247,7 +247,12 @@ typedef enum GraphOptimizationLevel { ORT_API_STATUS(OrtSetSessionGraphOptimizationLevel, _Inout_ OrtSessionOptions* options, GraphOptimizationLevel graph_optimization_level); -// How many threads in the session thread pool. +/** + * How many threads in the session thread pool. + * Set it to 0 to make onnxruntime run as single threaded. + * \param session_thread_pool_size <0, let the runtime choose a default. =0, Don't create extra threads. + * >0, create a thread pool with size of this value. + */ ORT_API_STATUS(OrtSetSessionThreadPoolSize, _Inout_ OrtSessionOptions* options, int session_thread_pool_size); /** diff --git a/onnxruntime/core/session/abi_session_options.cc b/onnxruntime/core/session/abi_session_options.cc index 647e5c3722a1..214c6b6818fe 100644 --- a/onnxruntime/core/session/abi_session_options.cc +++ b/onnxruntime/core/session/abi_session_options.cc @@ -129,8 +129,6 @@ ORT_API_STATUS_IMPL(OrtSetSessionGraphOptimizationLevel, _In_ OrtSessionOptions* ///How many threads in the session thread pool. ORT_API_STATUS_IMPL(OrtSetSessionThreadPoolSize, _In_ OrtSessionOptions* options, int session_thread_pool_size) { - if (session_thread_pool_size <= 0) - return OrtCreateStatus(ORT_INVALID_ARGUMENT, "session_thread_pool_size is not valid"); options->value.session_thread_pool_size = session_thread_pool_size; return nullptr; } diff --git a/onnxruntime/core/session/inference_session.cc b/onnxruntime/core/session/inference_session.cc index 6dc370b797f1..4d8fa7078409 100644 --- a/onnxruntime/core/session/inference_session.cc +++ b/onnxruntime/core/session/inference_session.cc @@ -92,8 +92,7 @@ inline std::basic_string GetCurrentTimeString() { } concurrency::ThreadPool* CreateThreadPool(int size) { - if (size <= 0) - size = std::thread::hardware_concurrency() / 2; + if (size < 0) size = std::thread::hardware_concurrency() / 2; return size > 0 ? new concurrency::ThreadPool("SESSION", size) : nullptr; } diff --git a/onnxruntime/core/session/inference_session.h b/onnxruntime/core/session/inference_session.h index 7fcbe0ad8713..632c92105bba 100644 --- a/onnxruntime/core/session/inference_session.h +++ b/onnxruntime/core/session/inference_session.h @@ -88,7 +88,7 @@ struct SessionOptions { TransformerLevel graph_optimization_level = TransformerLevel::Level1; // How many threads in the session thread pool. - int session_thread_pool_size = 0; + int session_thread_pool_size = -1; }; /** diff --git a/onnxruntime/test/perftest/command_args_parser.cc b/onnxruntime/test/perftest/command_args_parser.cc index ee54ee4b551d..7e70a956f622 100644 --- a/onnxruntime/test/perftest/command_args_parser.cc +++ b/onnxruntime/test/perftest/command_args_parser.cc @@ -32,7 +32,8 @@ namespace perftest { "\t-M: Disable memory pattern.\n" "\t-A: Disable memory arena\n" "\t-c [parallel runs]: Specifies the (max) number of runs to invoke simultaneously. Default:1.\n" - "\t-e [cpu|cuda|mkldnn|tensorrt|ngraph|openvino]: Specifies the provider 'cpu','cuda','mkldnn','tensorrt', 'ngraph' or 'openvino'. " + "\t-e [cpu|cuda|mkldnn|tensorrt|ngraph|openvino]: Specifies the provider 'cpu','cuda','mkldnn','tensorrt', " + "'ngraph' or 'openvino'. " "Default:'cpu'.\n" "\t-b [tf|ort]: backend to use. Default:ort\n" "\t-r [repeated_times]: Specifies the repeated times if running in 'times' test mode.Default:1000.\n" @@ -40,7 +41,7 @@ namespace perftest { "\t-p [profile_file]: Specifies the profile name to enable profiling and dump the profile data to the file.\n" "\t-s: Show statistics result, like P75, P90.\n" "\t-v: Show verbose information.\n" - "\t-x [thread_size]: Session thread pool size.\n" + "\t-x [thread_size]: Session thread pool size, must >=0.\n" "\t-P: Use parallel executor instead of sequential executor.\n" "\t-o [optimization level]: Please see onnxruntime_c_api.h (enum GraphOptimizationLevel) for the full list of all optimization levels. \n" "\t-h: help\n"); @@ -114,7 +115,7 @@ namespace perftest { break; case 'x': test_config.run_config.session_thread_pool_size = static_cast(OrtStrtol(optarg, nullptr)); - if (test_config.run_config.session_thread_pool_size <= 0) { + if (test_config.run_config.session_thread_pool_size < 0) { return false; } break; diff --git a/onnxruntime/test/perftest/ort_test_session.cc b/onnxruntime/test/perftest/ort_test_session.cc index d74fb1ba9ad3..f798cfaff730 100644 --- a/onnxruntime/test/perftest/ort_test_session.cc +++ b/onnxruntime/test/perftest/ort_test_session.cc @@ -91,9 +91,7 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device else session_options.DisableSequentialExecution(); fprintf(stdout, "Setting thread pool size to %d\n", performance_test_config.run_config.session_thread_pool_size); - // Don't set the thread pool size unless it has been changed from our zero default value (as zero will fail) - if (performance_test_config.run_config.session_thread_pool_size != 0) - session_options.SetThreadPoolSize(performance_test_config.run_config.session_thread_pool_size); + session_options.SetThreadPoolSize(performance_test_config.run_config.session_thread_pool_size); // Set optimization level. session_options.SetGraphOptimizationLevel(performance_test_config.run_config.optimization_level); if (!performance_test_config.run_config.profile_file.empty()) diff --git a/onnxruntime/test/perftest/test_configuration.h b/onnxruntime/test/perftest/test_configuration.h index 95f793bc9454..1ab5010948ec 100644 --- a/onnxruntime/test/perftest/test_configuration.h +++ b/onnxruntime/test/perftest/test_configuration.h @@ -44,7 +44,7 @@ struct RunConfig { bool enable_memory_pattern{true}; bool enable_cpu_mem_arena{true}; bool enable_sequential_execution{true}; - int session_thread_pool_size{0}; + int session_thread_pool_size{-1}; GraphOptimizationLevel optimization_level{ORT_ENABLE_EXTENDED}; };