diff --git a/csharp/sample/Microsoft.ML.OnnxRuntime.InferenceSample/Program.cs b/csharp/sample/Microsoft.ML.OnnxRuntime.InferenceSample/Program.cs index 0025bb8429f4..f794ed4bdae7 100644 --- a/csharp/sample/Microsoft.ML.OnnxRuntime.InferenceSample/Program.cs +++ b/csharp/sample/Microsoft.ML.OnnxRuntime.InferenceSample/Program.cs @@ -26,7 +26,7 @@ static void UseApi() // Optional : Create session options and set the graph optimization level for the session SessionOptions options = new SessionOptions(); - options.SetSessionGraphOptimizationLevel(2); + options.GraphOptimizationLevel = 2; using (var session = new InferenceSession(modelPath, options)) { diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/InferenceSession.cs b/csharp/src/Microsoft.ML.OnnxRuntime/InferenceSession.cs index 5f89bad8bbe9..d20a89b14c2a 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/InferenceSession.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/InferenceSession.cs @@ -19,6 +19,8 @@ public class InferenceSession : IDisposable { protected IntPtr _nativeHandle; protected Dictionary _inputMetadata, _outputMetadata; + private SessionOptions _builtInSessionOptions = null; + private RunOptions _builtInRunOptions = null; #region Public API @@ -28,10 +30,12 @@ public class InferenceSession : IDisposable /// /// public InferenceSession(string modelPath) - : this(modelPath, SessionOptions.Default) { + _builtInSessionOptions = new SessionOptions(); // need to be disposed + Init(modelPath, _builtInSessionOptions); } + /// /// Constructs an InferenceSession from a model file, with some additional session options /// @@ -39,52 +43,13 @@ public InferenceSession(string modelPath) /// public InferenceSession(string modelPath, SessionOptions options) { - var envHandle = OnnxRuntime.Handle; - - _nativeHandle = IntPtr.Zero; - try - { - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateSession(envHandle, System.Text.Encoding.Unicode.GetBytes(modelPath), options._nativePtr, out _nativeHandle)); - else - NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateSession(envHandle, System.Text.Encoding.UTF8.GetBytes(modelPath), options._nativePtr, out _nativeHandle)); - - // Initialize input/output metadata - _inputMetadata = new Dictionary(); - _outputMetadata = new Dictionary(); - - // get input count - UIntPtr inputCount = UIntPtr.Zero; - NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionGetInputCount(_nativeHandle, out inputCount)); - - // get all the output names - for (ulong i = 0; i < (ulong)inputCount; i++) - { - var iname = GetInputName(i); - _inputMetadata[iname] = GetInputMetadata(i); - } - // get output count - UIntPtr outputCount = UIntPtr.Zero; - NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionGetOutputCount(_nativeHandle, out outputCount)); - - // get all the output names - for (ulong i = 0; i < (ulong)outputCount; i++) - { - _outputMetadata[GetOutputName(i)] = GetOutputMetadata(i); - } - - } - catch (OnnxRuntimeException e) - { - if (_nativeHandle != IntPtr.Zero) - { - NativeMethods.OrtReleaseSession(_nativeHandle); - _nativeHandle = IntPtr.Zero; - } - throw e; - } + Init(modelPath, options); } + + /// + /// Meta data regarding the input nodes, keyed by input names + /// public IReadOnlyDictionary InputMetadata { get @@ -93,6 +58,9 @@ public IReadOnlyDictionary InputMetadata } } + /// + /// Metadata regarding the output nodes, keyed by output names + /// public IReadOnlyDictionary OutputMetadata { get @@ -101,11 +69,12 @@ public IReadOnlyDictionary OutputMetadata } } + /// /// Runs the loaded model for the given inputs, and fetches all the outputs. /// /// - /// Output Tensors in a Collection of NamedOnnxValue + /// Output Tensors in a Collection of NamedOnnxValue. User must dispose the output. public IDisposableReadOnlyCollection Run(IReadOnlyCollection inputs) { string[] outputNames = new string[_outputMetadata.Count]; @@ -118,21 +87,22 @@ public IDisposableReadOnlyCollection Run(IReadOnlyColl /// /// /// - /// Output Tensors in a Collection of NamedOnnxValue + /// Output Tensors in a Collection of NamedOnnxValue. User must dispose the output. public IDisposableReadOnlyCollection Run(IReadOnlyCollection inputs, IReadOnlyCollection outputNames) { - return Run(inputs, outputNames, RunOptions.Default); + IDisposableReadOnlyCollection result = null; + result = Run(inputs, outputNames, _builtInRunOptions); + return result; } /// - /// Runs the loaded model for the given inputs, and fetches the specified outputs in . + /// Runs the loaded model for the given inputs, and fetches the specified outputs in . /// /// /// /// - /// Output Tensors in a Collection of NamedOnnxValue - //TODO: kept internal until RunOptions is made public - internal IDisposableReadOnlyCollection Run(IReadOnlyCollection inputs, IReadOnlyCollection outputNames, RunOptions options) + /// Output Tensors in a Collection of NamedOnnxValue. User must dispose the output. + public IDisposableReadOnlyCollection Run(IReadOnlyCollection inputs, IReadOnlyCollection outputNames, RunOptions options) { var inputNames = new string[inputs.Count]; var inputTensors = new IntPtr[inputs.Count]; @@ -211,6 +181,58 @@ internal ModelMetadata ModelMetadata #endregion #region private methods + + protected void Init(string modelPath, SessionOptions options) + { + var envHandle = OnnxRuntime.Handle; + + _nativeHandle = IntPtr.Zero; + try + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateSession(envHandle, System.Text.Encoding.Unicode.GetBytes(modelPath), options.Handle, out _nativeHandle)); + else + NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateSession(envHandle, System.Text.Encoding.UTF8.GetBytes(modelPath), options.Handle, out _nativeHandle)); + + // Initialize input/output metadata + _inputMetadata = new Dictionary(); + _outputMetadata = new Dictionary(); + + // get input count + UIntPtr inputCount = UIntPtr.Zero; + NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionGetInputCount(_nativeHandle, out inputCount)); + + // get all the output names + for (ulong i = 0; i < (ulong)inputCount; i++) + { + var iname = GetInputName(i); + _inputMetadata[iname] = GetInputMetadata(i); + } + // get output count + UIntPtr outputCount = UIntPtr.Zero; + NativeApiStatus.VerifySuccess(NativeMethods.OrtSessionGetOutputCount(_nativeHandle, out outputCount)); + + // get all the output names + for (ulong i = 0; i < (ulong)outputCount; i++) + { + _outputMetadata[GetOutputName(i)] = GetOutputMetadata(i); + } + + } + catch (OnnxRuntimeException e) + { + if (_nativeHandle != IntPtr.Zero) + { + NativeMethods.OrtReleaseSession(_nativeHandle); + _nativeHandle = IntPtr.Zero; + } + throw e; + } + + _builtInRunOptions = new RunOptions(); // create a default built-in run option, and avoid creating a new one every run() call + } + + private string GetOutputName(ulong index) { IntPtr nameHandle = IntPtr.Zero; @@ -358,6 +380,15 @@ protected virtual void Dispose(bool disposing) if (disposing) { // cleanup managed resources + if (_builtInSessionOptions != null) + { + _builtInSessionOptions.Dispose(); + } + + if (_builtInRunOptions != null) + { + _builtInRunOptions.Dispose(); + } } // cleanup unmanaged resources @@ -426,24 +457,5 @@ internal class ModelMetadata //TODO: placeholder for Model metadata. Currently C-API does not expose this. } - /// Sets various runtime options. - /// TODO: currently uses Default options only. kept internal until fully implemented - internal class RunOptions - { - protected static readonly Lazy _default = new Lazy(() => new RunOptions()); - - public static RunOptions Default - { - get - { - return _default.Value; - } - } - - private void RuntOptions() - { - - } - } } diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs b/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs index 140516a6302f..5b9ed79559b7 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs @@ -184,6 +184,37 @@ IntPtr[] outputValues /* An array of output value pointers. Array must be alloca //[DllImport(nativeLib, CharSet = charSet)] //public static extern void OrtAddCustomOp(IntPtr /*(OrtSessionOptions*)*/ options, string custom_op_path); + #endregion + + #region RunOptions API + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /*(OrtStatus*)*/ OrtCreateRunOptions( out IntPtr /* OrtRunOptions** */ runOptions); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern void OrtReleaseRunOptions(IntPtr /*(OrtRunOptions*)*/options); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /*(OrtStatus*)*/ OrtRunOptionsSetRunLogVerbosityLevel(IntPtr /* OrtRunOptions* */ options, LogLevel value); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /*(OrtStatus*)*/ OrtRunOptionsSetRunTag(IntPtr /* OrtRunOptions* */ options, string /* const char* */ runTag); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /*(OrtStatus*)*/ OrtRunOptionsGetRunLogVerbosityLevel(IntPtr /* OrtRunOptions* */ options, out LogLevel verbosityLevel); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /*(OrtStatus*)*/ OrtRunOptionsGetRunTag(IntPtr /* const OrtRunOptions* */options, out IntPtr /* const char** */ runtag); + + // Set a flag so that any running OrtRun* calls that are using this instance of OrtRunOptions + // will exit as soon as possible if the flag is true. + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /*(OrtStatus*)*/ OrtRunOptionsEnableTerminate(IntPtr /* OrtRunOptions* */ options); + + [DllImport(nativeLib, CharSet = charSet)] + public static extern IntPtr /*(OrtStatus*)*/ OrtRunOptionsDisableTerminate(IntPtr /* OrtRunOptions* */ options); + + + #endregion #region Allocator/AllocatorInfo API diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/OnnxRuntime.cs b/csharp/src/Microsoft.ML.OnnxRuntime/OnnxRuntime.cs index 62141e9a8136..3069b4cb71b4 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/OnnxRuntime.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/OnnxRuntime.cs @@ -15,7 +15,7 @@ internal struct GlobalOptions //Options are currently not accessible to user public LogLevel LogLevel { get; set; } } - internal enum LogLevel + public enum LogLevel { Verbose = 0, Info = 1, @@ -51,6 +51,9 @@ public override bool IsInvalid private OnnxRuntime() //Problem: it is not possible to pass any option for a Singleton :base(IntPtr.Zero, true) { + // Check LibC version on Linux, before doing any onnxruntime initialization + CheckLibcVersionGreaterThanMinimum(); + handle = IntPtr.Zero; try { @@ -78,5 +81,32 @@ protected override bool ReleaseHandle() Delete(handle); return true; } + + [DllImport("libc", ExactSpelling = true, CallingConvention = CallingConvention.Cdecl)] + private static extern IntPtr gnu_get_libc_version(); + + private static void CheckLibcVersionGreaterThanMinimum() + { + // require libc version 2.23 or higher + var minVersion = new Version(2, 23); + var curVersion = new Version(0, 0); + if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) + { + try + { + curVersion = Version.Parse(Marshal.PtrToStringAnsi(gnu_get_libc_version())); + if (curVersion >= minVersion) + return; + } + catch (Exception) + { + // trap any obscure exception + } + throw new OnnxRuntimeException(ErrorCode.RuntimeException, + $"libc.so version={curVersion} does not meet the minimun of 2.23 required by OnnxRuntime. " + + "Linux distribution should be similar to Ubuntu 16.04 or higher"); + } + } + } } \ No newline at end of file diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/RunOptions.cs b/csharp/src/Microsoft.ML.OnnxRuntime/RunOptions.cs new file mode 100644 index 000000000000..6037503e70eb --- /dev/null +++ b/csharp/src/Microsoft.ML.OnnxRuntime/RunOptions.cs @@ -0,0 +1,112 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +using System; +using System.Runtime.InteropServices; + +namespace Microsoft.ML.OnnxRuntime +{ + /// Sets various runtime options. + public class RunOptions: IDisposable + { + private IntPtr _nativePtr; + + public RunOptions() + { + NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateRunOptions(out _nativePtr)); + } + + + /// + /// LogVerbosityLevel for the Run + /// default == LogLevel.Verbose + /// + public LogLevel LogVerbosityLevel + { + get + { + LogLevel level; + NativeApiStatus.VerifySuccess(NativeMethods.OrtRunOptionsGetRunLogVerbosityLevel(_nativePtr, out level)); + return level; + } + set + { + NativeApiStatus.VerifySuccess(NativeMethods.OrtRunOptionsSetRunLogVerbosityLevel(_nativePtr, value)); + } + } + + + /// + /// Log tag to be used during the run. default = "" + /// + public string LogTag + { + get + { + string tag = null; + IntPtr tagPtr = IntPtr.Zero; + NativeApiStatus.VerifySuccess(NativeMethods.OrtRunOptionsGetRunTag(_nativePtr, out tagPtr)); + tag = Marshal.PtrToStringAnsi(tagPtr); // assume ANSI string + // should not release the memory of the tagPtr, because it returns the c_str() of the std::string being used inside RunOptions C++ class + return tag; + } + set + { + NativeApiStatus.VerifySuccess(NativeMethods.OrtRunOptionsSetRunTag(_nativePtr, value)); + } + } + + + /// + /// Sets a flag to terminate any other Run() call that is using the same RunOptions object + /// Default = false + /// + public bool Terminate + { + get + { + return _terminate; + } + set + { + if (!_terminate && value) + { + NativeApiStatus.VerifySuccess(NativeMethods.OrtRunOptionsEnableTerminate(_nativePtr)); + _terminate = true; + } + else if (_terminate && !value) + { + NativeApiStatus.VerifySuccess(NativeMethods.OrtRunOptionsDisableTerminate(_nativePtr)); + _terminate = false; + } + } + } + private bool _terminate = false; //value set to default value of the C++ RunOptions + + + #region destructors disposers + + ~RunOptions() + { + Dispose(false); + } + + + public void Dispose() + { + GC.SuppressFinalize(this); + Dispose(true); + } + + + protected virtual void Dispose(bool disposing) + { + if (disposing) + { + // cleanup managed resources + } + NativeMethods.OrtReleaseRunOptions(_nativePtr); + } + + #endregion + } +} \ No newline at end of file diff --git a/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.cs b/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.cs index 13a334e0fd86..4ac90200f407 100644 --- a/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.cs +++ b/csharp/src/Microsoft.ML.OnnxRuntime/SessionOptions.cs @@ -4,126 +4,285 @@ using System; using System.Text; using System.Runtime.InteropServices; +using System.IO; namespace Microsoft.ML.OnnxRuntime { /// /// Holds the options for creating an InferenceSession /// - public class SessionOptions:IDisposable + public class SessionOptions : IDisposable { - public IntPtr _nativePtr; - protected static readonly Lazy _default = new Lazy(MakeSessionOptionWithCpuProvider); + private IntPtr _nativePtr; private static string[] cudaDelayLoadedLibs = { "cublas64_100.dll", "cudnn64_7.dll" }; + #region Constructor and Factory methods + /// /// Constructs an empty SessionOptions /// public SessionOptions() { - NativeMethods.OrtCreateSessionOptions(out _nativePtr); + NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateSessionOptions(out _nativePtr)); } + /// - /// Sets the graph optimization level for the session. Default is set to 1. + /// A helper method to constuct a SessionOptions object for CUDA execution /// - /// optimization level for the session - /// Available options are : 0, 1, 2 - /// 0 -> Disable all optimizations - /// 1 -> Enable basic optimizations - /// 2 -> Enable all optimizations - public void SetSessionGraphOptimizationLevel(uint optimization_level) + /// A SessionsOptions() object configured for execution on deviceId=0 + public static SessionOptions MakeSessionOptionWithCudaProvider() { - NativeApiStatus.VerifySuccess(NativeMethods.OrtSetSessionGraphOptimizationLevel(_nativePtr, optimization_level)); + return MakeSessionOptionWithCudaProvider(0); } + /// - /// Set filepath to save optimized model after graph level transformations. + /// A helper method to constuct a SessionOptions object for CUDA execution /// - /// File path for saving optimized model. - public void SetOptimizedModelFilePath(string optimizedModelFilepath) + /// + /// A SessionsOptions() object configured for execution on deviceId + public static SessionOptions MakeSessionOptionWithCudaProvider(int deviceId = 0) { - NativeApiStatus.VerifySuccess(NativeMethods.OrtSetOptimizedModelFilePath(_nativePtr, optimizedModelFilepath)); + CheckCudaExecutionProviderDLLs(); + SessionOptions options = new SessionOptions(); + NativeMethods.OrtSessionOptionsAppendExecutionProvider_CUDA(options._nativePtr, deviceId); + NativeMethods.OrtSessionOptionsAppendExecutionProvider_CPU(options._nativePtr, 1); + return options; + } + + #endregion + + #region Public Properties + + internal IntPtr Handle + { + get + { + return _nativePtr; + } } + /// - /// Enable Sequential Execution. By default, it is enabled. + /// Enable Sequential Execution. Default = true. /// /// - public void EnableSequentialExecution() + /// + public bool EnableSequentialExecution { - NativeApiStatus.VerifySuccess(NativeMethods.OrtEnableSequentialExecution(_nativePtr)); + get + { + return _enableSequentialExecution; + } + set + { + if (!_enableSequentialExecution && value) + { + NativeApiStatus.VerifySuccess(NativeMethods.OrtEnableSequentialExecution(_nativePtr)); + _enableSequentialExecution = true; + } + else if (_enableSequentialExecution && !value) + { + NativeApiStatus.VerifySuccess(NativeMethods.OrtDisableSequentialExecution(_nativePtr)); + _enableSequentialExecution = false; + } + } } + private bool _enableSequentialExecution = true; + /// - /// Disable Sequential Execution and enable Parallel Execution. + /// Enables the use of the memory allocation patterns in the first Run() call for subsequent runs. Default = true. /// - /// - public void DisableSequentialExecution() + public bool EnableMemoryPattern { - NativeApiStatus.VerifySuccess(NativeMethods.OrtDisableSequentialExecution(_nativePtr)); + get + { + return _enableMemoryPattern; + } + set + { + if (!_enableMemoryPattern && value) + { + NativeApiStatus.VerifySuccess(NativeMethods.OrtEnableMemPattern(_nativePtr)); + _enableMemoryPattern = true; + } + else if (_enableMemoryPattern && !value) + { + NativeApiStatus.VerifySuccess(NativeMethods.OrtDisableMemPattern(_nativePtr)); + _enableMemoryPattern = false; + } + } } + private bool _enableMemoryPattern = true; + /// - /// Enable Mem Pattern. By default, it is enabled + /// Path prefix to use for output of profiling data /// - /// - public void EnableMemPattern() + public string ProfileOutputPathPrefix + { + get; set; + } = "onnxruntime_profile_"; // this is the same default in C++ implementation + + + + /// + /// Enables profiling of InferenceSession.Run() calls. Default is false + /// + public bool EnableProfiling { - NativeApiStatus.VerifySuccess(NativeMethods.OrtEnableMemPattern(_nativePtr)); + get + { + return _enableProfiling; + } + set + { + if (!_enableProfiling && value) + { + NativeApiStatus.VerifySuccess(NativeMethods.OrtEnableProfiling(_nativePtr, ProfileOutputPathPrefix)); + _enableProfiling = true; + } + else if (_enableProfiling && !value) + { + NativeApiStatus.VerifySuccess(NativeMethods.OrtDisableProfiling(_nativePtr)); + _enableProfiling = false; + } + } } + private bool _enableProfiling = false; /// - /// Disable Mem Pattern. + /// Set filepath to save optimized model after graph level transformations. Default is empty, which implies saving is disabled. /// - /// - public void DisableMemPattern() + public string OptimizedModelFilePath { - NativeApiStatus.VerifySuccess(NativeMethods.OrtDisableMemPattern(_nativePtr)); + get + { + return _optimizedModelFilePath; + } + set + { + if (value != _optimizedModelFilePath) + { + NativeApiStatus.VerifySuccess(NativeMethods.OrtSetOptimizedModelFilePath(_nativePtr, value)); + _optimizedModelFilePath = value; + } + } } + private string _optimizedModelFilePath = ""; + + /// - /// Default instance + /// Enables Arena allocator for the CPU memory allocations. Default is true. /// - public static SessionOptions Default + public bool EnableCpuMemArena { get { - return _default.Value; + return _enableCpuMemArena; + } + set + { + if (!_enableCpuMemArena && value) + { + NativeApiStatus.VerifySuccess(NativeMethods.OrtEnableCpuMemArena(_nativePtr)); + _enableCpuMemArena = true; + } + else if (_enableCpuMemArena && !value) + { + NativeApiStatus.VerifySuccess(NativeMethods.OrtDisableCpuMemArena(_nativePtr)); + _enableCpuMemArena = false; + } } } + private bool _enableCpuMemArena = true; + - private static SessionOptions MakeSessionOptionWithCpuProvider() + /// + /// Log Id to be used for the session. Default is empty string. + /// TODO: Should it be named LogTag as in RunOptions? + /// + public string LogId { - CheckLibcVersionGreaterThanMinimum(); - SessionOptions options = new SessionOptions(); - NativeMethods.OrtSessionOptionsAppendExecutionProvider_CPU(options._nativePtr, 1); - return options; + get + { + return _logId; + } + + set + { + NativeApiStatus.VerifySuccess(NativeMethods.OrtSetSessionLogId(_nativePtr, value)); + _logId = value; + } } + private string _logId = ""; + /// - /// A helper method to constuct a SessionOptions object for CUDA execution + /// Log Verbosity Level for the session logs. Default = LogLevel.Verbose /// - /// A SessionsOptions() object configured for execution on deviceId=0 - public static SessionOptions MakeSessionOptionWithCudaProvider() + public LogLevel LogVerbosityLevel { - return MakeSessionOptionWithCudaProvider(0); + get + { + return _logVerbosityLevel; + } + set + { + NativeApiStatus.VerifySuccess(NativeMethods.OrtSetSessionLogVerbosityLevel(_nativePtr, value)); + _logVerbosityLevel = value; + } } + private LogLevel _logVerbosityLevel = LogLevel.Verbose; + /// - /// A helper method to constuct a SessionOptions object for CUDA execution + /// Threadpool size for the session.Run() calls. + /// Default = 0, meaning threadpool size is aumatically selected from number of available cores. /// - /// - /// A SessionsOptions() object configured for execution on deviceId - public static SessionOptions MakeSessionOptionWithCudaProvider(int deviceId=0) + public int ThreadPoolSize { - CheckLibcVersionGreaterThanMinimum(); - CheckCudaExecutionProviderDLLs(); - SessionOptions options = new SessionOptions(); - NativeMethods.OrtSessionOptionsAppendExecutionProvider_CUDA(options._nativePtr, deviceId); - NativeMethods.OrtSessionOptionsAppendExecutionProvider_CPU(options._nativePtr, 1); - return options; + get + { + return _threadPoolSize; + } + set + { + NativeApiStatus.VerifySuccess(NativeMethods.OrtSetSessionThreadPoolSize(_nativePtr, value)); + _threadPoolSize = value; + } + } + private int _threadPoolSize = 0; // set to what is set in C++ SessionOptions by default; + + + /// + /// Sets the graph optimization level for the session. Default is set to 1. + /// + /// Available options are : 0, 1, 2 + /// 0 -> Disable all optimizations + /// 1 -> Enable basic optimizations + /// 2 -> Enable all optimizations + public uint GraphOptimizationLevel + { + get + { + return _graphOptimizationLevel; + } + set + { + NativeApiStatus.VerifySuccess(NativeMethods.OrtSetSessionGraphOptimizationLevel(_nativePtr, value)); + _graphOptimizationLevel = value; + } } + private uint _graphOptimizationLevel = 1; + + #endregion + + #region Private Methods + // Declared, but called only if OS = Windows. [DllImport("kernel32.dll")] @@ -152,32 +311,8 @@ private static bool CheckCudaExecutionProviderDLLs() return true; } - [DllImport("libc", ExactSpelling = true, CallingConvention = CallingConvention.Cdecl)] - private static extern IntPtr gnu_get_libc_version(); - - private static void CheckLibcVersionGreaterThanMinimum() - { - // require libc version 2.23 or higher - var minVersion = new Version(2, 23); - var curVersion = new Version(0, 0); - if (RuntimeInformation.IsOSPlatform(OSPlatform.Linux)) - { - try - { - curVersion = Version.Parse(Marshal.PtrToStringAnsi(gnu_get_libc_version())); - if (curVersion >= minVersion) - return; - } - catch (Exception) - { - // trap any obscure exception - } - throw new OnnxRuntimeException(ErrorCode.RuntimeException, - $"libc.so version={curVersion} does not meet the minimun of 2.23 required by OnnxRuntime. " + - "Linux distribution should be similar to Ubuntu 16.04 or higher"); - } - } + #endregion #region destructors disposers ~SessionOptions() diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs index 471d7fb267ff..96ccddc97578 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs @@ -17,6 +17,84 @@ public class InferenceTest private const string module = "onnxruntime.dll"; private const string propertiesFile = "Properties.txt"; + [Fact] + + public void TestSessionOptions() + { + using (SessionOptions opt = new SessionOptions()) + { + Assert.NotNull(opt); + + // check default values of the properties + Assert.True(opt.EnableSequentialExecution); + Assert.True(opt.EnableMemoryPattern); + Assert.False(opt.EnableProfiling); + Assert.Equal("onnxruntime_profile_", opt.ProfileOutputPathPrefix); + Assert.True(opt.EnableCpuMemArena); + Assert.Equal("", opt.LogId); + Assert.Equal(LogLevel.Verbose, opt.LogVerbosityLevel); + Assert.Equal(0, opt.ThreadPoolSize); + Assert.Equal(1u, opt.GraphOptimizationLevel); + + // try setting options + opt.EnableSequentialExecution = false; + Assert.False(opt.EnableSequentialExecution); + + opt.EnableMemoryPattern = false; + Assert.False(opt.EnableMemoryPattern); + + opt.EnableProfiling = true; + Assert.True(opt.EnableProfiling); + Assert.Equal("onnxruntime_profile_", opt.ProfileOutputPathPrefix); + + opt.ProfileOutputPathPrefix = "Ort_P_"; + Assert.Equal("Ort_P_", opt.ProfileOutputPathPrefix); + + opt.EnableCpuMemArena = false; + Assert.False(opt.EnableCpuMemArena); + + opt.LogId = "MyLogId"; + Assert.Equal("MyLogId", opt.LogId); + + opt.LogVerbosityLevel = LogLevel.Error; + Assert.Equal(LogLevel.Error, opt.LogVerbosityLevel); + + opt.ThreadPoolSize = 4; + Assert.Equal(4, opt.ThreadPoolSize); + + opt.GraphOptimizationLevel = 3; + Assert.Equal(3u, opt.GraphOptimizationLevel); + + Assert.Throws(() => { opt.ThreadPoolSize = -2; }); + Assert.Throws(() => { opt.GraphOptimizationLevel = 10; }); + + } + } + + [Fact] + public void TestRunOptions() + { + using (var opt = new RunOptions()) + { + Assert.NotNull(opt); + + //verify default options + Assert.False(opt.Terminate); + Assert.Equal(LogLevel.Verbose, opt.LogVerbosityLevel); + Assert.Equal("", opt.LogTag); + + // try setting options + opt.Terminate = true; + Assert.True(opt.Terminate); + + opt.LogVerbosityLevel = LogLevel.Error; + Assert.Equal(LogLevel.Error, opt.LogVerbosityLevel); + + opt.LogTag = "MyLogTag"; + Assert.Equal("MyLogTag", opt.LogTag); + } + } + [Fact] public void CanCreateAndDisposeSessionWithModelPath() { @@ -61,8 +139,8 @@ private void CanRunInferenceOnAModel(uint graphOptimizationLevel, bool disableSe // Set the graph optimization level for this session. SessionOptions options = new SessionOptions(); - options.SetSessionGraphOptimizationLevel(graphOptimizationLevel); - if (disableSequentialExecution) options.DisableSequentialExecution(); + options.GraphOptimizationLevel = graphOptimizationLevel; + if (disableSequentialExecution) options.EnableSequentialExecution = false; using (var session = new InferenceSession(modelPath, options)) { @@ -82,32 +160,51 @@ private void CanRunInferenceOnAModel(uint graphOptimizationLevel, bool disableSe // Run the inference using (var results = session.Run(container)) // results is an IReadOnlyList container { - Assert.Equal(1, results.Count); + validateRunResults(results); + } + + // Run Inference with RunOptions + using (var runOptions = new RunOptions()) + { + runOptions.LogTag = "CsharpTest"; + runOptions.Terminate = true; + runOptions.LogVerbosityLevel = LogLevel.Error; + IReadOnlyCollection outputNames = session.OutputMetadata.Keys.ToList(); - float[] expectedOutput = LoadTensorFromFile(@"bench.expected_out"); - // validate the results - foreach (var r in results) + using (var results = session.Run(container, outputNames, runOptions)) // results is an IReadOnlyList container { - Assert.Equal("softmaxout_1", r.Name); + validateRunResults(results); + } + } + } + } - var resultTensor = r.AsTensor(); - int[] expectedDimensions = { 1, 1000, 1, 1 }; // hardcoded for now for the test data - Assert.Equal(expectedDimensions.Length, resultTensor.Rank); + private void validateRunResults(IDisposableReadOnlyCollection results) + { + float[] expectedOutput = LoadTensorFromFile(@"bench.expected_out"); + // validate the results + foreach (var r in results) + { + Assert.Equal(1, results.Count); + Assert.Equal("softmaxout_1", r.Name); - var resultDimensions = resultTensor.Dimensions; - for (int i = 0; i < expectedDimensions.Length; i++) - { - Assert.Equal(expectedDimensions[i], resultDimensions[i]); - } + var resultTensor = r.AsTensor(); + int[] expectedDimensions = { 1, 1000, 1, 1 }; // hardcoded for now for the test data + Assert.Equal(expectedDimensions.Length, resultTensor.Rank); - var resultArray = r.AsTensor().ToArray(); - Assert.Equal(expectedOutput.Length, resultArray.Length); - Assert.Equal(expectedOutput, resultArray, new floatComparer()); - } + var resultDimensions = resultTensor.Dimensions; + for (int i = 0; i < expectedDimensions.Length; i++) + { + Assert.Equal(expectedDimensions[i], resultDimensions[i]); } + + var resultArray = r.AsTensor().ToArray(); + Assert.Equal(expectedOutput.Length, resultArray.Length); + Assert.Equal(expectedOutput, resultArray, new floatComparer()); } } + [Fact] private void ThrowWrongInputName() { @@ -645,8 +742,8 @@ private void TestModelSerialization() string modelOutputPath = Path.Combine(Directory.GetCurrentDirectory(), "optimized-squeezenet.onnx"); // Set the optimized model file path to assert that no exception are thrown. SessionOptions options = new SessionOptions(); - options.SetOptimizedModelFilePath(modelOutputPath); - options.SetSessionGraphOptimizationLevel(1); + options.OptimizedModelFilePath = modelOutputPath; + options.GraphOptimizationLevel = 1; var session = new InferenceSession(modelPath, options); Assert.NotNull(session); Assert.True(File.Exists(modelOutputPath)); @@ -672,6 +769,7 @@ private void TestGpu() } } + [DllImport("kernel32", SetLastError = true)] static extern IntPtr LoadLibrary(string lpFileName); @@ -685,14 +783,19 @@ private void VerifyNativeMethodsExist() if (!RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) return; var entryPointNames = new[]{ - "OrtCreateEnv","OrtReleaseEnv","OrtGetErrorCode","OrtGetErrorMessage", - "OrtReleaseStatus","OrtCreateSession","OrtRun","OrtSessionGetInputCount", - "OrtSessionGetOutputCount","OrtSessionGetInputName","OrtSessionGetOutputName","OrtSessionGetInputTypeInfo", - "OrtSessionGetOutputTypeInfo","OrtReleaseSession","OrtCreateSessionOptions","OrtCloneSessionOptions", + "OrtCreateEnv","OrtReleaseEnv", + "OrtGetErrorCode","OrtGetErrorMessage", "OrtReleaseStatus", + "OrtCreateSession","OrtRun", + "OrtSessionGetInputCount", "OrtSessionGetOutputCount","OrtSessionGetInputName","OrtSessionGetOutputName", + "OrtSessionGetInputTypeInfo", "OrtSessionGetOutputTypeInfo","OrtReleaseSession", + "OrtCreateSessionOptions","OrtCloneSessionOptions", "OrtEnableSequentialExecution","OrtDisableSequentialExecution","OrtEnableProfiling","OrtDisableProfiling", "OrtEnableMemPattern","OrtDisableMemPattern","OrtEnableCpuMemArena","OrtDisableCpuMemArena", "OrtSetSessionLogId","OrtSetSessionLogVerbosityLevel","OrtSetSessionThreadPoolSize","OrtSetSessionGraphOptimizationLevel", - "OrtSetOptimizedModelFilePath", "OrtSessionOptionsAppendExecutionProvider_CPU","OrtCreateAllocatorInfo","OrtCreateCpuAllocatorInfo", + "OrtSetOptimizedModelFilePath", "OrtSessionOptionsAppendExecutionProvider_CPU", + "OrtCreateRunOptions", "OrtReleaseRunOptions", "OrtRunOptionsSetRunLogVerbosityLevel", "OrtRunOptionsSetRunTag", + "OrtRunOptionsGetRunLogVerbosityLevel", "OrtRunOptionsGetRunTag","OrtRunOptionsEnableTerminate", "OrtRunOptionsDisableTerminate", + "OrtCreateAllocatorInfo","OrtCreateCpuAllocatorInfo", "OrtCreateDefaultAllocator","OrtAllocatorFree","OrtAllocatorGetInfo", "OrtCreateTensorWithDataAsOrtValue","OrtGetTensorMutableData", "OrtReleaseAllocatorInfo", "OrtCastTypeInfoToTensorInfo","OrtGetTensorTypeAndShape","OrtGetTensorElementType","OrtGetDimensionsCount", diff --git a/csharp/tools/Microsoft.ML.OnnxRuntime.PerfTool/Program.cs b/csharp/tools/Microsoft.ML.OnnxRuntime.PerfTool/Program.cs index 0e9915ad6696..1764f79d19a9 100644 --- a/csharp/tools/Microsoft.ML.OnnxRuntime.PerfTool/Program.cs +++ b/csharp/tools/Microsoft.ML.OnnxRuntime.PerfTool/Program.cs @@ -93,8 +93,8 @@ static void RunModelOnnxRuntime(string modelPath, string inputPath, int iteratio timestamps[(int)TimingPoint.Start] = DateTime.Now; SessionOptions options = new SessionOptions(); - if (parallelExecution) options.DisableSequentialExecution(); - options.SetSessionGraphOptimizationLevel(optLevel); + if (parallelExecution) options.EnableSequentialExecution = false; + options.GraphOptimizationLevel = optLevel; using (var session = new InferenceSession(modelPath, options)) { timestamps[(int)TimingPoint.ModelLoaded] = DateTime.Now;