Merge branch 'gt/master' into gt/kedeng/imatmul

cmake/onnxruntime_mlas.cmake

@@ -17,13 +17,7 @@ set(mlas_common_srcs
 
 if(MSVC)
 
- if(CMAKE_GENERATOR_PLATFORM STREQUAL "ARM")
-
- set(mlas_platform_srcs
- ${ONNXRUNTIME_ROOT}/core/mlas/lib/arm/sgemmc.cpp
- )
-
- elseif(CMAKE_GENERATOR_PLATFORM STREQUAL "ARM64")
+ if(CMAKE_GENERATOR_PLATFORM STREQUAL "ARM64")
 
  set(asm_filename ${ONNXRUNTIME_ROOT}/core/mlas/lib/arm64/sgemma.asm)
  set(pre_filename ${CMAKE_CURRENT_BINARY_DIR}/sgemma.i)
@@ -45,17 +39,13 @@ if(MSVC)
 
  set(mlas_platform_srcs ${obj_filename})
 
- elseif(CMAKE_GENERATOR_PLATFORM STREQUAL "Win32")
-
- enable_language(ASM_MASM)
-
- set(CMAKE_ASM_MASM_FLAGS "${CMAKE_ASM_MASM_FLAGS} /safeseh")
+ elseif(CMAKE_GENERATOR_PLATFORM STREQUAL "ARM" OR CMAKE_GENERATOR MATCHES "ARM")
 
  set(mlas_platform_srcs
- ${ONNXRUNTIME_ROOT}/core/mlas/lib/i386/sgemma.asm
+ ${ONNXRUNTIME_ROOT}/core/mlas/lib/arm/sgemmc.cpp
  )
 
- elseif(CMAKE_GENERATOR_PLATFORM STREQUAL "x64")
+ elseif(CMAKE_GENERATOR_PLATFORM STREQUAL "x64" OR CMAKE_GENERATOR MATCHES "Win64")
 
  enable_language(ASM_MASM)
 
@@ -78,6 +68,16 @@ if(MSVC)
  ${ONNXRUNTIME_ROOT}/core/mlas/lib/amd64/ErfKernelFma3.asm
  )
 
+ else()
+
+ enable_language(ASM_MASM)
+
+ set(CMAKE_ASM_MASM_FLAGS "${CMAKE_ASM_MASM_FLAGS} /safeseh")
+
+ set(mlas_platform_srcs
+ ${ONNXRUNTIME_ROOT}/core/mlas/lib/i386/sgemma.asm
+ )
+
  endif()
 
 elseif(CMAKE_SYSTEM_NAME STREQUAL "Android")

cmake/onnxruntime_unittests.cmake

@@ -623,15 +623,6 @@ if (onnxruntime_BUILD_SHARED_LIB)
  protobuf::libprotobuf
  DEPENDS ${all_dependencies}
  )
- #demo
- message("PNG Lib Dir = ${PNG_LIBRARIES}")
- message("PNG Include Dir = ${PNG_INCLUDE_DIRS}")
- if(PNG_FOUND AND NOT WIN32) # for some reason some symbols are not found in Win32 PNG module
- add_executable(fns_candy_style_transfer "${ONNXRUNTIME_ROOT}/test/shared_lib/fns_candy_style_transfer.c")
- target_include_directories(fns_candy_style_transfer PRIVATE "${TEST_SRC_DIR}/util/include" ${PNG_INCLUDE_DIRS})
- target_link_libraries(fns_candy_style_transfer PRIVATE onnxruntime ${PNG_LIBRARIES})
- set_target_properties(fns_candy_style_transfer PROPERTIES FOLDER "ONNXRuntimeTest")
- endif()
 endif()
 
 if (onnxruntime_BUILD_SERVER)

include/onnxruntime/core/graph/graph.h

@@ -279,6 +279,10 @@ class Node {
  return !attr_to_subgraph_map_.empty();
  }
 
+ /** Get the const subgraphs from a node. 
+ @remarks Creates a new vector so calling ContainsSubgraphs first is preferred. */
+ std::vector<gsl::not_null<const Graph*>> GetSubgraphs() const;
+
  /** Gets a map of attribute name to the mutable Graph instances for all subgraphs of the Node.
  @returns Map of the attribute name that defines the subgraph to the subgraph's Graph instance.
  nullptr if the Node has no subgraphs.
@@ -756,6 +760,9 @@ class Graph {
  /** Returns the parent graph if this is a subgraph */
  const Graph* ParentGraph() const { return parent_graph_; }
 
+ /** Returns the mutable parent graph if this is a subgraph */
+ Graph* MutableParentGraph() { return parent_graph_; }
+
  /** Construct a Graph instance for a subgraph that is created from a GraphProto attribute in a Node.
  Inherits some properties from the parent graph.
  @param parent_graph The Graph containing the Node which has a GraphProto attribute.
@@ -980,6 +987,9 @@ class Graph {
  // NodeArgs that come from outer scope. Used when building a graph so that
  // these don't get recorded as graph inputs in the GraphProto.
  std::unordered_set<std::string> outer_scope_node_arg_names_;
+
+ // number of times Resolve has run.
+ int num_resolves_ = 0;
 };
 
 } // namespace onnxruntime

onnxruntime/contrib_ops/cpu/fused_activation.cc

@@ -0,0 +1,31 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#include "contrib_ops/cpu/fused_activation.h"
+
+namespace onnxruntime {
+
+common::Status GetFusedActivationAttr(const OpKernelInfo& info, MLAS_ACTIVATION& activation) {
+ // Convert the activation parameters from the node into a MLAS_ACTIVATION.
+ activation.ActivationKind = MlasIdentityActivation;
+
+ std::string activation_type;
+ if (info.GetAttr<std::string>("activation", &activation_type).IsOK()) {
+ if (activation_type == "Relu") {
+ activation.ActivationKind = MlasReluActivation;
+ } else if (activation_type == "LeakyRelu") {
+ activation.ActivationKind = MlasLeakyReluActivation;
+ activation.alpha = info.GetAttrOrDefault<float>("alpha", 0.01f);
+ } else if (activation_type == "Tanh") {
+ activation.ActivationKind = MlasTanhActivation;
+ } else if (activation_type == "Sigmoid") {
+ activation.ActivationKind = MlasLogisticActivation;
+ } else {
+ return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "Unimplemented activation: " + activation_type);
+ }
+ }
+
+ return Status::OK();
+}
+
+} // namespace onnxruntime

onnxruntime/contrib_ops/cpu/fused_activation.h

@@ -0,0 +1,14 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+#include "core/common/common.h"
+#include "core/framework/op_kernel.h"
+#include "core/util/math.h"
+#include "core/mlas/inc/mlas.h"
+
+namespace onnxruntime {
+
+common::Status GetFusedActivationAttr(const OpKernelInfo& info, MLAS_ACTIVATION& activation);
+
+} // namespace onnxruntime

onnxruntime/contrib_ops/cpu/fused_conv.cc

@@ -1,16 +1,26 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
-#include "fused_conv.h"
+#include "core/providers/cpu/nn/conv.h"
+#include "contrib_ops/cpu/fused_activation.h"
 
 namespace onnxruntime {
 namespace contrib {
+
+class FusedConvFloat final : public Conv<float> {
+ public:
+ FusedConvFloat(const OpKernelInfo& info) : Conv<float>(info) {
+ ORT_ENFORCE(GetFusedActivationAttr(info, activation_).IsOK());
+ }
+};
+
 ONNX_CPU_OPERATOR_TYPED_MS_KERNEL(
  FusedConv,
  1,
  float,
  KernelDefBuilder()
  .TypeConstraint("T", DataTypeImpl::GetTensorType<float>()),
- FusedConv<float>);
+ FusedConvFloat);
+
 } // namespace contrib
 } // namespace onnxruntime

onnxruntime/contrib_ops/cpu/fused_gemm.cc

@@ -1,15 +1,26 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
-#include "fused_gemm.h"
+#include "core/providers/cpu/math/gemm.h"
 
 namespace onnxruntime {
 namespace contrib {
+
+template <typename T>
+class FusedGemm final : public Gemm<T> {
+ public:
+ FusedGemm(const OpKernelInfo& info) : Gemm<T>(info) {
+ Gemm<T>::activation_ = info.GetAttrOrDefault<std::string>("activation", "");
+ Gemm<T>::leaky_relu_alpha_ = info.GetAttrOrDefault("leaky_relu_alpha", 0.01f);
+ }
+};
+
 ONNX_CPU_OPERATOR_TYPED_MS_KERNEL(
  FusedGemm,
  1,
  float,
  KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<float>()),
- FusedGemm<float, float, float, float>);
+ FusedGemm<float>);
+
 } // namespace contrib
 } // namespace onnxruntime

onnxruntime/contrib_ops/cpu/nchwc_ops.cc

@@ -34,7 +34,7 @@ ONNX_CPU_OPERATOR_TYPED_NCHWC_KERNEL(
  KernelDefBuilder()
  .MayInplace(3, 0)
  .TypeConstraint("T", DataTypeImpl::GetTensorType<float>()),
- NchwcConv<float>);
+ NchwcConv);
 
 ONNX_CPU_OPERATOR_TYPED_NCHWC_KERNEL(
  MaxPool,
@@ -70,39 +70,38 @@ ONNX_CPU_OPERATOR_TYPED_NCHWC_KERNEL(
 
 template <typename T>
 Status ReorderInput<T>::Compute(OpKernelContext* context) const {
- const Tensor* X = context->Input<Tensor>(0);
- const TensorShape& X_shape = X->Shape();
+ const auto* X = context->Input<Tensor>(0);
+ const auto& X_shape = X->Shape();
  ORT_ENFORCE(X_shape.NumDimensions() == 4);
  ORT_ENFORCE((X_shape[1] % MlasNchwcGetBlockSize()) == 0);
- Tensor* Y = context->Output(0, X_shape);
+ auto* Y = context->Output(0, X_shape);
  MlasReorderInput(X_shape.GetDims().data(), X->template Data<T>(), Y->template MutableData<T>());
  return Status::OK();
 }
 
 template <typename T>
 Status ReorderOutput<T>::Compute(OpKernelContext* context) const {
- const Tensor* X = context->Input<Tensor>(0);
- const TensorShape& X_shape = X->Shape();
+ const auto* X = context->Input<Tensor>(0);
+ const auto& X_shape = X->Shape();
  ORT_ENFORCE(X_shape.NumDimensions() == 4);
  std::vector<int64_t> Y_shape(X_shape.GetDims());
  ORT_ENFORCE(channels_ <= Y_shape[1]);
  Y_shape[1] = channels_;
- Tensor* Y = context->Output(0, Y_shape);
+ auto* Y = context->Output(0, Y_shape);
  MlasReorderOutput(Y_shape.data(), X->template Data<T>(), Y->template MutableData<T>());
  return Status::OK();
 }
 
-template <typename T>
-Status NchwcConv<T>::Compute(OpKernelContext* context) const {
- const Tensor* X = context->Input<Tensor>(0);
- const Tensor* W = context->Input<Tensor>(1);
- const Tensor* B = context->Input<Tensor>(2);
- const Tensor* Sum = context->Input<Tensor>(3);
+Status NchwcConv::Compute(OpKernelContext* context) const {
+ const auto* X = context->Input<Tensor>(0);
+ const auto* W = context->Input<Tensor>(1);
+ const auto* B = context->Input<Tensor>(2);
+ const auto* Sum = context->Input<Tensor>(3);
 
  ORT_RETURN_IF_ERROR(ConvBase::ValidateInputShape(X, W));
 
- const TensorShape& X_shape = X->Shape();
- const TensorShape& W_shape = W->Shape();
+ const auto& X_shape = X->Shape();
+ const auto& W_shape = W->Shape();
  ORT_ENFORCE(X_shape.NumDimensions() == 4);
 
  const size_t nchwc_block_size = MlasNchwcGetBlockSize();
@@ -131,36 +130,20 @@ Status NchwcConv<T>::Compute(OpKernelContext* context) const {
  Y_dims.insert(Y_dims.begin(), {X_shape[0], W_shape[0]});
  TensorShape input_shape = X->Shape().Slice(2);
  ORT_RETURN_IF_ERROR(ConvBase::InferOutputShape(input_shape, kernel_shape, strides, dilations, &pads, &Y_dims));
- Tensor* Y = context->Output(0, Y_dims);
- T* y_data = Y->template MutableData<T>();
+ auto* Y = context->Output(0, Y_dims);
+ auto* y_data = Y->template MutableData<float>();
 
  // Check for the optional Conv/Sum fusion.
  if (Sum != nullptr) {
  const auto& sum_shape = Sum->Shape();
  ORT_RETURN_IF_NOT(Y->Shape() == sum_shape, "output and sum shape must match");
  // If the output was not allocated inplace with the sum tensor, then copy here.
- const float* sum_data = Sum->template Data<T>();
+ const auto* sum_data = Sum->template Data<float>();
  if (y_data != sum_data) {
- memcpy(y_data, sum_data, sum_shape.Size() * sizeof(T));
+ memcpy(y_data, sum_data, sum_shape.Size() * sizeof(float));
  }
  }
 
- MLAS_ACTIVATION Activation;
- if (ConvBase::activation_.empty()) {
- Activation.ActivationKind = MlasIdentityActivation;
- } else if (ConvBase::activation_ == "Relu") {
- Activation.ActivationKind = MlasReluActivation;
- } else if (ConvBase::activation_ == "LeakyRelu") {
- Activation.ActivationKind = MlasLeakyReluActivation;
- Activation.alpha = ConvBase::alpha_;
- } else if (ConvBase::activation_ == "Tanh") {
- Activation.ActivationKind = MlasTanhActivation;
- } else if (ConvBase::activation_ == "Sigmoid") {
- Activation.ActivationKind = MlasLogisticActivation;
- } else {
- ORT_NOT_IMPLEMENTED("Not implemented fused activation: ", ConvBase::activation_);
- }
-
  MlasNchwcConv(kernel_shape.size(),
  X_shape.GetDims().data(),
  kernel_shape.data(),
@@ -173,17 +156,17 @@ Status NchwcConv<T>::Compute(OpKernelContext* context) const {
  W->template Data<float>(),
  B != nullptr ? B->template Data<float>() : nullptr,
  y_data,
- &Activation,
+ &activation_,
  Sum == nullptr,
  const_cast<concurrency::ThreadPool*>(static_cast<OpKernelContextInternal*>(context)->GetOperatorThreadPool()));
 
  return Status::OK();
 }
 
 Status NchwcPoolBase::NchwcPool(OpKernelContext* context, MLAS_POOLING_KIND kind) const {
- const Tensor* X = context->Input<Tensor>(0);
+ const auto* X = context->Input<Tensor>(0);
 
- const TensorShape& X_shape = X->Shape();
+ const auto& X_shape = X->Shape();
  ORT_ENFORCE(X_shape.NumDimensions() == 4);
  ORT_ENFORCE((X_shape[1] % MlasNchwcGetBlockSize()) == 0);
 
@@ -193,7 +176,7 @@ Status NchwcPoolBase::NchwcPool(OpKernelContext* context, MLAS_POOLING_KIND kind
 
  std::vector<int64_t> pads = pads_;
  std::vector<int64_t> output_dims = PoolBase::SetOutputSize(X_shape, X_shape[1], &pads, dilations_, ceil_mode_);
- Tensor* Y = context->Output(0, output_dims);
+ auto* Y = context->Output(0, output_dims);
 
  MlasNchwcPool(kind,
  2,