Allow user disable multiple threading

csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs

@@ -63,8 +63,7 @@ public void TestSessionOptions()
 
  opt.GraphOptimizationLevel = GraphOptimizationLevel.ORT_ENABLE_EXTENDED;
  Assert.Equal(GraphOptimizationLevel.ORT_ENABLE_EXTENDED, opt.GraphOptimizationLevel);
-
- Assert.Throws<OnnxRuntimeException>(() => { opt.ThreadPoolSize = -2; });
+
  Assert.Throws<OnnxRuntimeException>(() => { opt.GraphOptimizationLevel = (GraphOptimizationLevel)10; });
  }
  }

include/onnxruntime/core/session/onnxruntime_c_api.h

@@ -247,7 +247,12 @@ typedef enum GraphOptimizationLevel {
 ORT_API_STATUS(OrtSetSessionGraphOptimizationLevel, _Inout_ OrtSessionOptions* options,
  GraphOptimizationLevel graph_optimization_level);
 
-// How many threads in the session thread pool.
+/**
+ * How many threads in the session thread pool.
+ * Set it to 0 to make onnxruntime run as single threaded.
+ * \param session_thread_pool_size <0, let the runtime choose a default. =0, Don't create extra threads. 
+ * >0, create a thread pool with size of this value.
+ */
 ORT_API_STATUS(OrtSetSessionThreadPoolSize, _Inout_ OrtSessionOptions* options, int session_thread_pool_size);
 
 /**

onnxruntime/core/session/abi_session_options.cc

@@ -129,8 +129,6 @@ ORT_API_STATUS_IMPL(OrtSetSessionGraphOptimizationLevel, _In_ OrtSessionOptions*
 
 ///How many threads in the session thread pool.
 ORT_API_STATUS_IMPL(OrtSetSessionThreadPoolSize, _In_ OrtSessionOptions* options, int session_thread_pool_size) {
- if (session_thread_pool_size <= 0)
- return OrtCreateStatus(ORT_INVALID_ARGUMENT, "session_thread_pool_size is not valid");
  options->value.session_thread_pool_size = session_thread_pool_size;
  return nullptr;
 }

onnxruntime/core/session/inference_session.cc

@@ -92,8 +92,7 @@ inline std::basic_string<T> GetCurrentTimeString() {
 }
 
 concurrency::ThreadPool* CreateThreadPool(int size) {
- if (size <= 0)
- size = std::thread::hardware_concurrency() / 2;
+ if (size < 0) size = std::thread::hardware_concurrency() / 2;
  return size > 0 ? new concurrency::ThreadPool("SESSION", size) : nullptr;
 }
 

onnxruntime/core/session/inference_session.h

@@ -88,7 +88,7 @@ struct SessionOptions {
  TransformerLevel graph_optimization_level = TransformerLevel::Level1;
 
  // How many threads in the session thread pool.
- int session_thread_pool_size = 0;
+ int session_thread_pool_size = -1;
 };
 
 /**

onnxruntime/test/perftest/command_args_parser.cc

@@ -32,15 +32,16 @@ namespace perftest {
  "\t-M: Disable memory pattern.\n"
  "\t-A: Disable memory arena\n"
  "\t-c [parallel runs]: Specifies the (max) number of runs to invoke simultaneously. Default:1.\n"
- "\t-e [cpu|cuda|mkldnn|tensorrt|ngraph|openvino]: Specifies the provider 'cpu','cuda','mkldnn','tensorrt', 'ngraph' or 'openvino'. "
+ "\t-e [cpu|cuda|mkldnn|tensorrt|ngraph|openvino]: Specifies the provider 'cpu','cuda','mkldnn','tensorrt', "
+ "'ngraph' or 'openvino'. "
  "Default:'cpu'.\n"
  "\t-b [tf|ort]: backend to use. Default:ort\n"
  "\t-r [repeated_times]: Specifies the repeated times if running in 'times' test mode.Default:1000.\n"
  "\t-t [seconds_to_run]: Specifies the seconds to run for 'duration' mode. Default:600.\n"
  "\t-p [profile_file]: Specifies the profile name to enable profiling and dump the profile data to the file.\n"
  "\t-s: Show statistics result, like P75, P90.\n"
  "\t-v: Show verbose information.\n"
- "\t-x [thread_size]: Session thread pool size.\n"
+ "\t-x [thread_size]: Session thread pool size, must >=0.\n"
  "\t-P: Use parallel executor instead of sequential executor.\n"
  "\t-o [optimization level]: Please see onnxruntime_c_api.h (enum GraphOptimizationLevel) for the full list of all optimization levels. \n"
  "\t-h: help\n");
@@ -114,7 +115,7 @@ namespace perftest {
  break;
  case 'x':
  test_config.run_config.session_thread_pool_size = static_cast<int>(OrtStrtol<PATH_CHAR_TYPE>(optarg, nullptr));
- if (test_config.run_config.session_thread_pool_size <= 0) {
+ if (test_config.run_config.session_thread_pool_size < 0) {
  return false;
  }
  break;

onnxruntime/test/perftest/ort_test_session.cc

@@ -91,9 +91,7 @@ OnnxRuntimeTestSession::OnnxRuntimeTestSession(Ort::Env& env, std::random_device
  else
  session_options.DisableSequentialExecution();
  fprintf(stdout, "Setting thread pool size to %d\n", performance_test_config.run_config.session_thread_pool_size);
- // Don't set the thread pool size unless it has been changed from our zero default value (as zero will fail)
- if (performance_test_config.run_config.session_thread_pool_size != 0)
- session_options.SetThreadPoolSize(performance_test_config.run_config.session_thread_pool_size);
+ session_options.SetThreadPoolSize(performance_test_config.run_config.session_thread_pool_size);
  // Set optimization level.
  session_options.SetGraphOptimizationLevel(performance_test_config.run_config.optimization_level);
  if (!performance_test_config.run_config.profile_file.empty())

onnxruntime/test/perftest/test_configuration.h

@@ -44,7 +44,7 @@ struct RunConfig {
  bool enable_memory_pattern{true};
  bool enable_cpu_mem_arena{true};
  bool enable_sequential_execution{true};
- int session_thread_pool_size{0};
+ int session_thread_pool_size{-1};
  GraphOptimizationLevel optimization_level{ORT_ENABLE_EXTENDED};
 };