Merge branch 'master' of https://github.com/microsoft/onnxruntime int…

…o transpose

cgmanifests/submodules/cgmanifest.json

@@ -345,7 +345,7 @@
       "component": {
         "type": "git",
         "git": {
-          "commitHash": "b71de776d2847b6e0fed54a888a8213a7812ad12",
+          "commitHash": "a7a0fec7f25cae567429af62b7eaaee1c3f0e247",
           "repositoryUrl": "https://github.com/onnx/onnx"
         },
         "comments": "git submodule at cmake/external/onnx"

cmake/CMakeLists.txt

@@ -62,7 +62,6 @@ option(onnxruntime_USE_EIGEN_FOR_BLAS "Use eign for blas" ON)
 option(onnxruntime_USE_NNAPI_BUILTIN "Build with builtin NNAPI lib for Android NNAPI support" OFF)
 option(onnxruntime_USE_RKNPU "Build with RKNPU support" OFF)
 option(onnxruntime_USE_DNNL "Build with DNNL support" OFF)
-option(onnxruntime_USE_MKLML "Build the default cpu provider with MKL-ML binary dependency" OFF)
 option(onnxruntime_USE_FEATURIZERS "Build ML Featurizers support" OFF)
 option(onnxruntime_USE_NGRAPH "Build with nGraph support" OFF)
 option(onnxruntime_USE_OPENBLAS "Use openblas" OFF)
@@ -198,12 +197,7 @@ if(onnxruntime_USE_OPENMP)
   if (OPENMP_FOUND)
     set (CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${OpenMP_C_FLAGS}")
     set (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${OpenMP_CXX_FLAGS}")
-    include_directories(${OpenMP_CXX_INCLUDE_DIR})
-    # MKLML and NGraph depend on their own OpenMP library that may be different with the compiler's.
-    # Disable the options to build mklml/NGraph and OpenMP together.
-    if((WIN32 OR APPLE) AND onnxruntime_USE_MKLML)
-      message(FATAL_ERROR "Please use only one of onnxruntime_USE_MKLML, onnxruntime_USE_OPENMP")
-    endif()
+    include_directories(${OpenMP_CXX_INCLUDE_DIR})    
     if(onnxruntime_USE_NGRAPH)
       message(FATAL_ERROR "Please use only one of onnxruntime_USE_NGRAPH, onnxruntime_USE_OPENMP")
     endif()
@@ -696,8 +690,7 @@ if (onnxruntime_USE_ARMNN)
   list(APPEND onnxruntime_EXTERNAL_LIBRARIES armnn pthread arm_compute_core arm_compute arm_compute_graph)
 endif()
 
-# MKLML
-if (onnxruntime_USE_DNNL OR onnxruntime_USE_MKLML)
+if (onnxruntime_USE_DNNL)
   include(dnnl)
 endif()
 
@@ -713,29 +706,13 @@ if (onnxruntime_USE_TVM)
   if (onnxruntime_USE_OPENMP)
     set(USE_OPENMP "gnu")
   endif()
-  if (onnxruntime_USE_MKLML)
-    set(USE_OPENMP "intel")
-    # make sure MKLML in ORT is used by TVM
-    if (WIN32)
-      set(OMP_LIBRARY ${MKLML_LIB_DIR}/${IOMP5MD_IMPORT_LIB})
-    else()
-      set(OMP_LIBRARY ${MKLML_LIB_DIR}/${IOMP5MD_SHARED_LIB})
-    endif()
-  endif()
 
   add_subdirectory(${PROJECT_SOURCE_DIR}/external/tvm EXCLUDE_FROM_ALL)
   set_target_properties(tvm PROPERTIES FOLDER "External/tvm")
   set_target_properties(tvm_topi PROPERTIES FOLDER "External/tvm")
   set_target_properties(tvm_runtime PROPERTIES FOLDER "External/tvm")
   set_target_properties(nnvm_compiler PROPERTIES FOLDER "External/tvm")
 
-  if (onnxruntime_USE_MKLML)
-    add_dependencies(tvm project_mklml)
-    add_dependencies(tvm_topi project_mklml)
-    add_dependencies(tvm_runtime project_mklml)
-    add_dependencies(nnvm_compiler project_mklml)
-  endif()
-
   set(TVM_INCLUDES ${PROJECT_SOURCE_DIR}/external/tvm/include
     ${PROJECT_SOURCE_DIR}/external/tvm/3rdparty/dmlc-core/include
     ${PROJECT_SOURCE_DIR}/external/tvm/3rdparty/dlpack/include
@@ -784,22 +761,6 @@ endif()
 set_target_properties(onnx PROPERTIES FOLDER "External/ONNX")
 set_target_properties(onnx_proto PROPERTIES FOLDER "External/ONNX")
 
-if (onnxruntime_USE_MKLML)
-  add_definitions(-DUSE_MKLML=1 -DUSE_MKLML_FOR_BLAS=1)
-  if (WIN32 OR APPLE)
-    list(APPEND onnxruntime_EXTERNAL_LIBRARIES mklml)
-  else()
-    if(onnxruntime_USE_OPENMP)
-      list(APPEND onnxruntime_EXTERNAL_LIBRARIES mklml_gnu)
-    else()
-      list(APPEND onnxruntime_EXTERNAL_LIBRARIES mklml_intel)
-    endif()
-  endif()
-  list(APPEND onnxruntime_EXTERNAL_DEPENDENCIES project_mklml)
-  include_directories(${MKLML_INCLUDE_DIR})
-  link_directories(${MKLML_LIB_DIR})
-endif()
-
 if (onnxruntime_USE_NGRAPH)
   if (NOT onnxruntime_USE_FULL_PROTOBUF)
     message(FATAL_ERROR "Please set onnxruntime_USE_FULL_PROTOBUF=ON for nGraph execution provider.")

cmake/external/dnnl.cmake

@@ -1,64 +1,25 @@
 include (ExternalProject)
 
 set(DNNL_URL https://github.com/intel/mkl-dnn.git)
-# If DNNL_TAG is updated, check if MKLML_VERSION and platform.cmake.patch need to be updated.
+# If DNNL_TAG is updated, check if platform.cmake.patch need to be updated.
 set(DNNL_TAG v1.1.1)
-set(MKLML_VERSION 2019.0.5.20190502)
 
 if(WIN32)
   set(MKLML_OS_VERSION_STR "win")
   set(MKLML_FILE_EXTENSION "zip")
   set(DNNL_SHARED_LIB dnnl.dll)
-  set(DNNL_IMPORT_LIB dnnl.lib)
-  if(onnxruntime_USE_MKLML)
-    # Windows-only updated MKLML binary which contains fix for thread cleanup hang.
-    set(MKLML_VERSION 2020.0.20190813)
-    set(MKLML_SHARED_LIB mklml.dll)
-    set(MKLML_IMPORT_LIB mklml.lib)
-    set(IOMP5MD_SHARED_LIB libiomp5md.dll)
-    set(IOMP5MD_IMPORT_LIB libiomp5md.lib)
-  endif()
+  set(DNNL_IMPORT_LIB dnnl.lib)  
 else()
   set(MKLML_FILE_EXTENSION "tgz")
   if (APPLE)
     set(DNNL_SHARED_LIB libdnnl.1.dylib)
-    set(MKLML_OS_VERSION_STR "mac")
-    if(onnxruntime_USE_MKLML)
-      set(MKLML_SHARED_LIB libmklml.dylib)
-      set(IOMP5MD_SHARED_LIB libiomp5.dylib)
-    endif()
+    set(MKLML_OS_VERSION_STR "mac")    
   else()
     set(DNNL_SHARED_LIB libdnnl.so.1)
-    set(MKLML_OS_VERSION_STR "lnx")
-    if(onnxruntime_USE_MKLML)
-      if(onnxruntime_USE_OPENMP)
-        set(MKLML_SHARED_LIB libmklml_gnu.so)
-      else()
-        set(MKLML_SHARED_LIB libmklml_intel.so)
-        set(IOMP5MD_SHARED_LIB libiomp5.so)
-      endif()
-    endif()
+    set(MKLML_OS_VERSION_STR "lnx")    
   endif()  
 endif()
 
-if (onnxruntime_USE_MKLML)
-  set(MKLDNN_VERSION_SHORT v0.20)
-  set(MKLML_URL https://github.com/intel/mkl-dnn/releases/download/${MKLDNN_VERSION_SHORT}/mklml_${MKLML_OS_VERSION_STR}_${MKLML_VERSION}.${MKLML_FILE_EXTENSION})
-
-  ExternalProject_Add(project_mklml
-    PREFIX mklml
-    URL ${MKLML_URL}
-    CONFIGURE_COMMAND ""
-    BUILD_COMMAND ""
-    UPDATE_COMMAND ""
-    INSTALL_COMMAND ""  )
-
-  set(MKML_DIR ${CMAKE_CURRENT_BINARY_DIR}/mklml/src/project_mklml)
-  set(MKLML_INCLUDE_DIR "${MKML_DIR}/include")
-  set(MKLML_LIB_DIR "${MKML_DIR}/lib")
-  link_directories(${MKLML_LIB_DIR})
-endif()
-
 if (onnxruntime_USE_DNNL)
   set(DNNL_SOURCE ${CMAKE_CURRENT_BINARY_DIR}/dnnl/src/dnnl/src)
   set(DNNL_INSTALL ${CMAKE_CURRENT_BINARY_DIR}/dnnl/install)
@@ -88,7 +49,4 @@ if (onnxruntime_USE_DNNL)
     CMAKE_ARGS -DDNNL_BUILD_TESTS=OFF -DDNNL_BUILD_EXAMPLES=OFF -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -DCMAKE_INSTALL_PREFIX=${DNNL_INSTALL}
   )
   link_directories(${DNNL_LIB_DIR})
-  #if (onnxruntime_USE_MKLML)
-  #  add_dependencies(project_dnnl project_mklml)
-  #endif()
 endif()

cmake/onnxruntime_mlas.cmake

@@ -9,6 +9,7 @@ set(mlas_common_srcs
   ${ONNXRUNTIME_ROOT}/core/mlas/lib/qgemm.cpp
   ${ONNXRUNTIME_ROOT}/core/mlas/lib/convolve.cpp
   ${ONNXRUNTIME_ROOT}/core/mlas/lib/pooling.cpp
+  ${ONNXRUNTIME_ROOT}/core/mlas/lib/transpose.cpp
   ${ONNXRUNTIME_ROOT}/core/mlas/lib/reorder.cpp
   ${ONNXRUNTIME_ROOT}/core/mlas/lib/snchwc.cpp
   ${ONNXRUNTIME_ROOT}/core/mlas/lib/activate.cpp

cmake/onnxruntime_python.cmake

@@ -378,15 +378,6 @@ if (onnxruntime_USE_TVM)
   )
 endif()
 
-if (onnxruntime_USE_MKLML)
-  add_custom_command(
-    TARGET onnxruntime_pybind11_state POST_BUILD
-    COMMAND ${CMAKE_COMMAND} -E copy
-        ${MKLML_LIB_DIR}/${MKLML_SHARED_LIB} ${MKLML_LIB_DIR}/${IOMP5MD_SHARED_LIB}
-        $<TARGET_FILE_DIR:${test_data_target}>/onnxruntime/capi/
-  )
-endif()
-
 if (onnxruntime_USE_NUPHAR)
   file(GLOB onnxruntime_python_nuphar_python_srcs CONFIGURE_DEPENDS
     "${ONNXRUNTIME_ROOT}/core/providers/nuphar/scripts/*"

cmake/onnxruntime_unittests.cmake

@@ -645,14 +645,6 @@ if (onnxruntime_USE_DNNL)
     COMMAND ${CMAKE_COMMAND} -E copy ${DNNL_DLL_PATH} $<TARGET_FILE_DIR:${test_data_target}>
     )
 endif()
-if (onnxruntime_USE_MKLML)
-  add_custom_command(
-    TARGET ${test_data_target} POST_BUILD
-    COMMAND ${CMAKE_COMMAND} -E copy
-    ${MKLML_LIB_DIR}/${MKLML_SHARED_LIB} ${MKLML_LIB_DIR}/${IOMP5MD_SHARED_LIB}
-    $<TARGET_FILE_DIR:${test_data_target}>
-  )
-endif()
 if(WIN32)
   if (onnxruntime_USE_NGRAPH)
     add_custom_command(

csharp/src/Microsoft.ML.OnnxRuntime/InferenceSession.cs

@@ -676,7 +676,7 @@ public ulong ProfilingStartTimeNs
 
         private void Init(string modelPath, SessionOptions options)
         {
-            var envHandle = OnnxRuntime.Handle;
+            var envHandle = OrtEnv.Handle;
             var session = IntPtr.Zero;
             NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateSession(envHandle, NativeMethods.GetPlatformSerializedString(modelPath), options.Handle, out session));
 
@@ -685,7 +685,7 @@ private void Init(string modelPath, SessionOptions options)
 
         private void Init(byte[] modelData, SessionOptions options)
         {
-            var envHandle = OnnxRuntime.Handle;
+            var envHandle = OrtEnv.Handle;
             var session = IntPtr.Zero;
 
             NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateSessionFromArray(envHandle, modelData, (UIntPtr)modelData.Length, options.Handle, out session));

csharp/src/Microsoft.ML.OnnxRuntime/NativeMethods.cs

@@ -202,6 +202,9 @@ static NativeMethods()
 
             OrtCreateEnv = (DOrtCreateEnv)Marshal.GetDelegateForFunctionPointer(api_.CreateEnv, typeof(DOrtCreateEnv));
             OrtReleaseEnv = (DOrtReleaseEnv)Marshal.GetDelegateForFunctionPointer(api_.ReleaseEnv, typeof(DOrtReleaseEnv));
+            OrtEnableTelemetryEvents = (DOrtEnableTelemetryEvents)Marshal.GetDelegateForFunctionPointer(api_.EnableTelemetryEvents, typeof(DOrtEnableTelemetryEvents));
+            OrtDisableTelemetryEvents = (DOrtDisableTelemetryEvents)Marshal.GetDelegateForFunctionPointer(api_.DisableTelemetryEvents, typeof(DOrtDisableTelemetryEvents));
+
             OrtGetErrorCode = (DOrtGetErrorCode)Marshal.GetDelegateForFunctionPointer(api_.GetErrorCode, typeof(DOrtGetErrorCode));
             OrtGetErrorMessage = (DOrtGetErrorMessage)Marshal.GetDelegateForFunctionPointer(api_.GetErrorMessage, typeof(DOrtGetErrorMessage));
             OrtReleaseStatus = (DOrtReleaseStatus)Marshal.GetDelegateForFunctionPointer(api_.ReleaseStatus, typeof(DOrtReleaseStatus));
@@ -332,6 +335,12 @@ static NativeMethods()
         public delegate void DOrtReleaseEnv(IntPtr /*(OrtEnv*)*/ env);
         public static DOrtReleaseEnv OrtReleaseEnv;
 
+        public delegate IntPtr /* OrtStatus* */DOrtEnableTelemetryEvents(IntPtr /*(OrtEnv*)*/ env);
+        public static DOrtEnableTelemetryEvents OrtEnableTelemetryEvents;
+
+        public delegate IntPtr /* OrtStatus* */DOrtDisableTelemetryEvents(IntPtr /*(OrtEnv*)*/ env);
+        public static DOrtDisableTelemetryEvents OrtDisableTelemetryEvents;
+
         #endregion Runtime/Environment API
 
         #region Status API

csharp/src/Microsoft.ML.OnnxRuntime/OnnxRuntime.cs

@@ -38,41 +38,77 @@ public enum OrtLanguageProjection
     }
 
     /// <summary>
-    /// This class intializes the process-global ONNX runtime
-    /// C# API users do not need to access this, thus kept as internal
+    /// This class initializes the process-global ONNX Runtime environment instance (OrtEnv)
     /// </summary>
-    internal sealed class OnnxRuntime : SafeHandle
+    public sealed class OrtEnv : SafeHandle
     {
-        private static readonly Lazy<OnnxRuntime> _instance = new Lazy<OnnxRuntime>(()=> new OnnxRuntime());
-
-        internal static IntPtr Handle  // May throw exception in every access, if the constructor have thrown an exception
+        private static readonly Lazy<OrtEnv> _instance = new Lazy<OrtEnv>(()=> new OrtEnv());
+
+        #region private methods
+        private OrtEnv()  //Problem: it is not possible to pass any option for a Singleton
+    : base(IntPtr.Zero, true)
         {
-            get
+            NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateEnv(LogLevel.Warning, @"CSharpOnnxRuntime", out handle));
+            try
             {
-                return _instance.Value.handle;
+                NativeApiStatus.VerifySuccess(NativeMethods.OrtSetLanguageProjection(handle, OrtLanguageProjection.ORT_PROJECTION_CSHARP));
+            }
+            catch (OnnxRuntimeException e)
+            {
+                ReleaseHandle();
+                throw e;
             }
         }
+        #endregion
 
-        public override bool IsInvalid
+        #region internal methods
+        /// <summary>
+        /// Returns a handle to the native `OrtEnv` instance held by the singleton C# `OrtEnv` instance
+        /// Exception caching: May throw an exception on every call, if the `OrtEnv` constructor threw an exception
+        /// during lazy initialization
+        /// </summary>
+        internal static IntPtr Handle  
         {
             get
             {
-                return (handle == IntPtr.Zero);
+                return _instance.Value.handle;
             }
         }
+        #endregion
+
+        #region public methods
 
-        private OnnxRuntime()  //Problem: it is not possible to pass any option for a Singleton
-            :base(IntPtr.Zero, true)
+        /// <summary>
+        /// Returns an instance of OrtEnv
+        /// It returns the same instance on every call - `OrtEnv` is singleton
+        /// </summary>
+        public static OrtEnv Instance() { return _instance.Value; }
+
+        /// <summary>
+        /// Enable platform telemetry collection where applicable
+        /// (currently only official Windows ORT builds have telemetry collection capabilities)
+        /// </summary>
+        public void EnableTelemetryEvents()
         {
-            NativeApiStatus.VerifySuccess(NativeMethods.OrtCreateEnv(LogLevel.Warning, @"CSharpOnnxRuntime", out handle));
-            try
-            {
-                NativeApiStatus.VerifySuccess(NativeMethods.OrtSetLanguageProjection(handle, OrtLanguageProjection.ORT_PROJECTION_CSHARP));
-            }
-            catch (OnnxRuntimeException e)
+            NativeApiStatus.VerifySuccess(NativeMethods.OrtEnableTelemetryEvents(Handle));
+        }
+
+        /// <summary>
+        /// Disable platform telemetry collection
+        /// </summary>
+        public void DisableTelemetryEvents()
+        {
+            NativeApiStatus.VerifySuccess(NativeMethods.OrtDisableTelemetryEvents(Handle));
+        }
+
+        #endregion
+
+        #region SafeHandle
+        public override bool IsInvalid
+        {
+            get
             {
-                ReleaseHandle();
-                throw e;
+                return (handle == IntPtr.Zero);
             }
         }
 
@@ -82,5 +118,6 @@ protected override bool ReleaseHandle()
             handle = IntPtr.Zero;
             return true;
         }
+        #endregion
     }
 }

csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker-gpu.sh

@@ -27,6 +27,6 @@ docker run --gpus all --rm \
         -e "PackageName=$PackageName" \
         -e "RunTestCsharp=$RunTestCsharp" \
         -e "RunTestNative=$RunTestNative" \
-        onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecentosgpubuild:ch5h \
+        onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecentosgpubuild:ch9q \
         /bin/bash /onnxruntime_src/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.sh \
         /home/onnxruntimedev/$NUGET_REPO_DIRNAME /onnxruntime_src /home/onnxruntimedev $CurrentOnnxRuntimeVersion

csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker.sh

@@ -35,6 +35,6 @@ docker run --rm \
         -e "DisableMlOps=$DISABLEMLOPS" \
         -e "RunTestCsharp=$RunTestCsharp" \
         -e "RunTestNative=$RunTestNative" \
-        onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecentoscpubuild:ch5g \
+        onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecentoscpubuild:ch9m \
         /bin/bash /onnxruntime_src/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.sh \
         /home/onnxruntimedev/$NUGET_REPO_DIRNAME /onnxruntime_src /home/onnxruntimedev $CurrentOnnxRuntimeVersion

csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs

@@ -164,6 +164,18 @@ public void TestRunOptions()
             }
         }
 
+        [Fact]
+        public void EnablingAndDisablingTelemetryEventCollection()
+        {
+            var ortEnvInstance = OrtEnv.Instance();
+            ortEnvInstance.DisableTelemetryEvents();
+
+            // no-op on non-Windows builds
+            // may be no-op on certain Windows builds based on build configuration
+
+            ortEnvInstance.EnableTelemetryEvents();
+        }
+
         [Fact]
         public void CanCreateAndDisposeSessionWithModelPath()
         {