Standalone TVM Executor Provider (microsoft#10019)

* squashed commit for standalone tvm execution provider * critical fix for correct python build with stvm ep * get tuning log file from ep options. It has priority over AUTOTVM_TUNING_LOG * updates and fixes * update parsing of stvm provider options * add support of external data for onnx model * add conditional dump of subgraphs * remove unused code * get input tensor shapes through provider options. get output shapes for fixed input ones by TVM API * support AUTO_TVM tuning log file inside ORT. Selector for Ansor and Auto_TVM is provider option (tuning_type) * add fp16 * add functionality of conversion of model layout to NHWC if need. Necessary parameter was added to STVM provider options * fix license text in header. fix log format * small fixes * fix issues from flake8 * remove model proto construction from GetCapability * reserve memory for vector of DLTensors * add simple tutorial for STVM EP * STVM docs * jroesch/tvm -> apache/tvm * remove dead code, unneccessary logs and comments * fix in readme * improve tutorial notebook * tvm update * update STVM_EP.md * fix default value * update STVM_EP.md * some TODOs for the future development * shorten long lines * add hyperlink to STVM_EP.md * fix Linux CI error * fix error in csharp test Co-authored-by: Jared Roesch <jroesch@octoml.ai> Co-authored-by: Valery Chernov <valery.chernov@deelvin.com> Co-authored-by: KJlaccHoeUM9l <wotpricol@mail.ru> (cherry picked from commit b327e89)
octoml · Nov 7, 2022 · 39269f7 · 39269f7
1 parent 0d9030e
commit 39269f7
Show file tree

Hide file tree

Showing 48 changed files with 2,251 additions and 7 deletions.
diff --git a/.gitignore b/.gitignore
@@ -15,6 +15,7 @@ gen
 *~
 .vs
 .DS_Store
+*.DS_Store
 TestResults/
 .idea/
 onnxruntime.egg-info

diff --git a/.gitmodules b/.gitmodules
@@ -79,3 +79,6 @@
 [submodule "cmake/external/onnx-tensorrt"]
  path = cmake/external/onnx-tensorrt
  url = https://github.com/onnx/onnx-tensorrt.git
+[submodule "cmake/external/tvm_update"]
+ path = cmake/external/tvm_update
+ url = https://github.com/apache/tvm
diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt
@@ -59,8 +59,8 @@ option(onnxruntime_BUILD_OBJC "Build Objective-C library" OFF)
 option(onnxruntime_USE_PREINSTALLED_EIGEN "Use pre-installed EIGEN. Need to provide eigen_SOURCE_PATH if turn this on." OFF)
 option(onnxruntime_BUILD_BENCHMARKS "Build ONNXRuntime micro-benchmarks" OFF)
 
-option(onnxruntime_USE_TVM "Build tvm for code-gen" OFF)
-option(onnxruntime_USE_LLVM "Build tvm with LLVM" OFF)
+option(onnxruntime_USE_TVM "Build TVM for code generation." OFF)
+option(onnxruntime_USE_LLVM "Build TVM with LLVM" OFF)
 
 option(onnxruntime_BUILD_FOR_NATIVE_MACHINE "Enable this option for turning on optimization specific to this machine" OFF)
 option(onnxruntime_USE_AVX "Use AVX instructions" OFF)
@@ -103,6 +103,7 @@ option(onnxruntime_USE_TELEMETRY "Build with Telemetry" OFF)
 #It's default OFF because it's experimental now.
 option(onnxruntime_PREFER_SYSTEM_LIB "Experimental: Build with the preinstalled libraries in your system" OFF)
 option(onnxruntime_USE_ROCM "Build with AMD GPU support" OFF)
+option(onnxruntime_USE_STVM "Build with standalone TVM support" OFF)
 
 # Options related to reducing the binary size produced by the build
 option(onnxruntime_DISABLE_CONTRIB_OPS "Disable contrib ops" OFF)
@@ -1102,6 +1103,11 @@ if (onnxruntime_USE_NUPHAR)
  list(APPEND ORT_PROVIDER_CMAKE_FLAGS -Donnxruntime_USE_NUPHAR=1)
  list(APPEND ONNXRUNTIME_PROVIDER_NAMES nuphar)
 endif()
+if (onnxruntime_USE_STVM)
+ list(APPEND ORT_PROVIDER_FLAGS -DUSE_STVM=1)
+ list(APPEND ORT_PROVIDER_CMAKE_FLAGS -Donnxruntime_USE_STVM=1)
+ list(APPEND ONNXRUNTIME_PROVIDER_NAMES stvm)
+endif()
 if (onnxruntime_USE_WINML)
  list(APPEND ORT_PROVIDER_FLAGS -DUSE_WINML=1)
  list(APPEND ORT_PROVIDER_CMAKE_FLAGS -Donnxruntime_USE_WINML=1)

diff --git a/cmake/external/tvm_update b/cmake/external/tvm_update
diff --git a/cmake/onnxruntime.cmake b/cmake/onnxruntime.cmake
@@ -167,6 +167,7 @@ set(onnxruntime_INTERNAL_LIBRARIES
  ${PROVIDERS_MIGRAPHX}
  ${PROVIDERS_NNAPI}
  ${PROVIDERS_NUPHAR}
+ ${PROVIDERS_STVM}
  ${PROVIDERS_RKNPU}
  ${PROVIDERS_ROCM}
  ${PROVIDERS_VITISAI}

diff --git a/cmake/onnxruntime_csharp.cmake b/cmake/onnxruntime_csharp.cmake
@@ -34,6 +34,10 @@ if (onnxruntime_USE_NUPHAR)
  STRING(APPEND CSHARP_PREPROCESSOR_DEFINES "USE_NUPHAR;")
 endif()
 
+if (onnxruntime_USE_STVM)
+ STRING(APPEND CSHARP_PREPROCESSOR_DEFINES "USE_STVM,")
+endif()
+
 if (onnxruntime_USE_OPENVINO)
  STRING(APPEND CSHARP_PREPROCESSOR_DEFINES "USE_OPENVINO;")
 endif()

diff --git a/cmake/onnxruntime_java.cmake b/cmake/onnxruntime_java.cmake
@@ -62,7 +62,6 @@ file(GLOB onnxruntime4j_native_src
 onnxruntime_add_shared_library_module(onnxruntime4j_jni ${onnxruntime4j_native_src})
 set_property(TARGET onnxruntime4j_jni PROPERTY CXX_STANDARD 11)
 
-
 # depend on java sources. if they change, the JNI should recompile
 add_dependencies(onnxruntime4j_jni onnxruntime4j)
 onnxruntime_add_include_to_target(onnxruntime4j_jni onnxruntime_session)

diff --git a/cmake/onnxruntime_providers.cmake b/cmake/onnxruntime_providers.cmake
@@ -92,6 +92,10 @@ endif()
 if(onnxruntime_USE_ROCM)
  set(PROVIDERS_ROCM onnxruntime_providers_rocm)
 endif()
+if (onnxruntime_USE_STVM)
+ set(PROVIDERS_STVM onnxruntime_providers_stvm)
+endif()
+
 
 source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_common_srcs} ${onnxruntime_providers_srcs})
 
@@ -1160,3 +1164,46 @@ if (onnxruntime_USE_ROCM)
  RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR})
 
 endif()
+
+if (onnxruntime_USE_STVM)
+ add_definitions(-DUSE_STVM=1)
+
+ file (GLOB_RECURSE onnxruntime_providers_stvm_cc_srcs CONFIGURE_DEPENDS
+ "${ONNXRUNTIME_ROOT}/core/providers/stvm/*.h"
+ "${ONNXRUNTIME_ROOT}/core/providers/stvm/*.cc"
+ )
+ source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_stvm_cc_srcs})
+ onnxruntime_add_static_library(onnxruntime_providers_stvm ${onnxruntime_providers_stvm_cc_srcs})
+
+ if ( CMAKE_COMPILER_IS_GNUCC )
+ target_compile_options(onnxruntime_providers_stvm PRIVATE -Wno-unused-parameter -Wno-missing-field-initializers)
+ endif()
+
+ target_include_directories(onnxruntime_providers_stvm PRIVATE
+ ${onnxruntime_STVM_HOME}/include
+ ${onnxruntime_STVM_HOME}/3rdparty/dlpack/include
+ ${onnxruntime_STVM_HOME}/3rdparty/dmlc-core/include
+ ${PYTHON_INLCUDE_DIRS})
+ onnxruntime_add_include_to_target(onnxruntime_providers_stvm onnxruntime_common onnx)
+
+ add_dependencies(onnxruntime_providers_stvm ${onnxruntime_EXTERNAL_DEPENDENCIES})
+
+ target_link_libraries(onnxruntime_providers_stvm PRIVATE
+ onnx
+ onnxruntime_common
+ onnxruntime_framework
+ )
+ if(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+ target_link_libraries(onnxruntime_providers_stvm PRIVATE ${onnxruntime_STVM_HOME}/build/libtvm.dylib)
+ else()
+ target_link_libraries(onnxruntime_providers_stvm PRIVATE ${onnxruntime_STVM_HOME}/build/libtvm.so)
+ endif()
+
+ set_target_properties(onnxruntime_providers_stvm PROPERTIES FOLDER "ONNXRuntime")
+ set_target_properties(onnxruntime_providers_stvm PROPERTIES LINKER_LANGUAGE CXX)
+
+ target_compile_options(onnxruntime_providers_stvm PRIVATE -Wno-error=type-limits)
+ target_compile_definitions(onnxruntime_providers_stvm PUBLIC DMLC_USE_LOGGING_LIBRARY=<tvm/runtime/logging.h>)
+
+ install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/stvm DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers)
+endif()
diff --git a/cmake/onnxruntime_python.cmake b/cmake/onnxruntime_python.cmake
@@ -138,6 +138,7 @@ target_link_libraries(onnxruntime_pybind11_state PRIVATE
  ${onnxruntime_libs}
  ${PROVIDERS_MIGRAPHX}
  ${PROVIDERS_NUPHAR}
+ ${PROVIDERS_STVM}
  ${PROVIDERS_VITISAI}
  ${PROVIDERS_NNAPI}
  ${PROVIDERS_COREML}
@@ -654,6 +655,23 @@ if (onnxruntime_USE_NUPHAR)
  )
 endif()
 
+if (onnxruntime_USE_STVM)
+ file(GLOB onnxruntime_python_providers_stvm_srcs CONFIGURE_DEPENDS
+ "${ONNXRUNTIME_ROOT}/python/providers/stvm/*.py"
+ )
+ add_custom_command(
+ TARGET onnxruntime_pybind11_state POST_BUILD
+ COMMAND ${CMAKE_COMMAND} -E make_directory $<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/providers
+ COMMAND ${CMAKE_COMMAND} -E make_directory $<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/providers/stvm
+ COMMAND ${CMAKE_COMMAND} -E copy
+ ${onnxruntime_python_providers_stvm_srcs}
+ $<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/providers/stvm
+ COMMAND ${CMAKE_COMMAND} -E copy
+ $<TARGET_FILE:onnxruntime_providers_stvm>
+ $<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/capi/
+ )
+endif()
+
 if (onnxruntime_USE_DML)
  add_custom_command(
  TARGET onnxruntime_pybind11_state POST_BUILD

diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake
@@ -483,6 +483,7 @@ set(ONNXRUNTIME_TEST_LIBS
  ${PROVIDERS_ACL}
  ${PROVIDERS_ARMNN}
  ${PROVIDERS_COREML}
+ # ${PROVIDERS_STVM}
  onnxruntime_optimizer
  onnxruntime_providers
  onnxruntime_util
@@ -537,6 +538,14 @@ if(onnxruntime_USE_COREML)
  endif()
 endif()
 
+if (onnxruntime_USE_STVM)
+ file (GLOB_RECURSE onnxruntime_test_stvm_src CONFIGURE_DEPENDS
+ "${ONNXRUNTIME_ROOT}/test/stvm/*.h"
+ "${ONNXRUNTIME_ROOT}/test/stvm/*.cc"
+ )
+
+ list(APPEND onnxruntime_test_providers_dependencies onnxruntime_providers_stvm)
+endif()
 
 if(WIN32)
  if (onnxruntime_USE_TVM)
@@ -626,6 +635,11 @@ endif()
 if (onnxruntime_USE_TVM)
  list(APPEND all_tests ${onnxruntime_test_tvm_src})
 endif()
+
+if (onnxruntime_USE_STVM)
+ list(APPEND all_tests ${onnxruntime_test_stvm_src})
+endif()
+
 if (onnxruntime_USE_OPENVINO)
  list(APPEND all_tests ${onnxruntime_test_openvino_src})
 endif()
@@ -1205,4 +1219,12 @@ if (NOT onnxruntime_MINIMAL_BUILD AND NOT onnxruntime_EXTENDED_MINIMAL_BUILD
  endif()
 endif()
 
+if (onnxruntime_USE_STVM)
+ # find_library(STVM_LIBS NAMES libtvm.so PATHS ${onnxruntime_STVM_HOME}/lib)
+ # link_directories(onnxruntime_test_all ${STVM_LIBS})
+ find_library(PYTHON_LIBS NAMES libpython PATHS /usr/local/lib)
+ #target_link_libraries(onnxruntime_test_all PRIVATE ${PYTHON_LIBRARIES} -lutil)
+ # set(CMAKE_SHARED_LINKER_FLAGS "-Wl,-rpath,${STVM_LIBS}")
+endif()
+
 include(onnxruntime_fuzz_test.cmake)
diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.shared.cs b/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.shared.cs
@@ -792,6 +792,9 @@ IntPtr[] outputValues /* An array of output value pointers. Array must be alloca
 
  [DllImport(NativeLib.DllName, CharSet = CharSet.Ansi)]
  public static extern IntPtr /*(OrtStatus*)*/ OrtSessionOptionsAppendExecutionProvider_Nuphar(IntPtr /*(OrtSessionOptions*) */ options, int allow_unaligned_buffers, IntPtr /*(char char*)*/ settings);
+
+ [DllImport(NativeLib.DllName, CharSet = CharSet.Ansi)]
+ public static extern IntPtr /*(OrtStatus*)*/ OrtSessionOptionsAppendExecutionProvider_Stvm(IntPtr /*(OrtSessionOptions*) */ options, IntPtr /*(char char*)*/ settings);
 #endif
  /// <summary>
  /// Append a TensorRT EP instance (configured based on given provider options) to the native OrtSessionOptions instance

diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.shared.cs b/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.shared.cs
@@ -127,6 +127,20 @@ public static SessionOptions MakeSessionOptionWithNupharProvider(String settings
  return options;
  }
 
+ /// <summary>
+ /// A helper method to construct a SessionOptions object for Stvm execution.
+ /// Use only if you have the onnxruntime package specific to this Execution Provider.
+ /// </summary>
+ /// <param name="settings">settings string, comprises of comma separated key:value pairs. default is empty</param>
+ /// <returns>A SessionsOptions() object configured for execution with Stvm</returns>
+ public static SessionOptions MakeSessionOptionWithStvmProvider(String settings = "")
+ {
+ SessionOptions options = new SessionOptions();
+ options.AppendExecutionProvider_Stvm(settings);
+
+ return options;
+ }
+
  /// <summary>
  /// A helper method to construct a SessionOptions object for ROCM execution.
  /// Use only if ROCM is installed and you have the onnxruntime package specific to this Execution Provider.
@@ -304,6 +318,23 @@ public void AppendExecutionProvider_Nuphar(string settings = "")
  {
  NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionOptionsAppendExecutionProvider_Nuphar(handle, 1, pinnedSettingsName.Pointer));
  }
+#endif
+ }
+
+ /// <summary>
+ /// Use only if you have the onnxruntime package specific to this Execution Provider.
+ /// </summary>
+ /// <param name="settings">string with Stvm specific settings</param>
+ public void AppendExecutionProvider_Stvm(string settings = "")
+ {
+#if __MOBILE__
+ throw new NotSupportedException("The Stvm Execution Provider is not supported in this build");
+#else
+ var settingsPinned = GCHandle.Alloc(NativeOnnxValueHelper.StringToZeroTerminatedUtf8(settings), GCHandleType.Pinned);
+ using (var pinnedSettingsName = new PinnedGCHandle(settingsPinned))
+ {
+ NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionOptionsAppendExecutionProvider_Stvm(handle, pinnedSettingsName.Pointer));
+ }
 #endif
  }
 #endregion //ExecutionProviderAppends

diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/InferenceTest.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Common/InferenceTest.cs
@@ -118,6 +118,10 @@ public void TestSessionOptions()
  opt.AppendExecutionProvider_Nnapi(0);
 #endif
 
+#if USE_STVM
+ opt.AppendExecutionProvider_Stvm("Vulkan -device=amd_apu");
+#endif 
+
 #if USE_NUPHAR
  opt.AppendExecutionProvider_Nuphar();
 #endif