Merge pull request microsoft#8 from chenfeiyue-cfy/vsinpu

Vsinpu

onnxruntime/core/providers/vsinpu/builders/impl/base_op_builder.cc

@@ -59,12 +59,8 @@ bool BaseOpBuilder::IsSupported(const onnxruntime::GraphViewer& graph_viewer,
  return false;
  }
 
- if (node->Domain() != "") {
- LOGS_DEFAULT(VERBOSE) << "Only support node with default domain!";
- return false;
- }
-
  if (!util::CheckNoZeroDim(node)) {
+ LOGS_DEFAULT(VERBOSE) << "Dynamic shape(shape has zero dim) is not supported!";
  return false;
  }
 

onnxruntime/core/providers/vsinpu/builders/impl/clip_op_builder.cc

@@ -0,0 +1,112 @@
+/****************************************************************************
+ *
+ * Copyright (c) 2024 Vivante Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ *****************************************************************************/
+#include "core/providers/vsinpu/builders/impl/clip_op_builder.h"
+
+namespace onnxruntime {
+namespace vsi {
+namespace npu {
+
+namespace clip_internal {
+template <typename T>
+struct LowMax {
+ constexpr static T low() {
+ return std::numeric_limits<T>::lowest();
+ }
+ constexpr static T max() {
+ return std::numeric_limits<T>::max();
+ }
+};
+} // namespace clip_internal
+
+template <typename T>
+struct ClipOpBuilder::ClipImpl {
+ ClipImpl(vsi::npu::GraphEP* graph_ep, std::vector<std::shared_ptr<tim::vx::Tensor>>& inputs,
+ std::vector<std::shared_ptr<tim::vx::Tensor>>& outputs) {
+ T min_default = clip_internal::LowMax<T>::low();
+ T max_default = clip_internal::LowMax<T>::max();
+
+ T* min_data = &min_default;
+ T* max_data = &max_default;
+ std::shared_ptr<tim::vx::Tensor> min_tensor = nullptr;
+ std::shared_ptr<tim::vx::Tensor> max_tensor = nullptr;
+ if (inputs.size() > 1) {
+ min_tensor = inputs[1];
+ if (inputs.size() > 2) {
+ max_tensor = inputs[2];
+ }
+ }
+ if (min_tensor) {
+ min_tensor->CopyDataFromTensor(min_data);
+ }
+ if (max_tensor) {
+ max_tensor->CopyDataFromTensor(max_data);
+ }
+ auto op = graph_ep->GetGraph()->CreateOperation<tim::vx::ops::Clip>(static_cast<float>(*min_data), static_cast<float>(*max_data));
+ (*op).BindInputs(inputs).BindOutputs(outputs);
+ graph_ep->GetOps().push_back(std::move(op));
+ }
+};
+
+bool ClipOpBuilder::HandleBuildOp(vsi::npu::GraphEP* graph_ep,
+ std::vector<std::shared_ptr<tim::vx::Tensor>>& inputs,
+ std::vector<std::shared_ptr<tim::vx::Tensor>>& outputs,
+ const Node* node) {
+ LOGS_DEFAULT(INFO) << "Creating Clip Op.";
+ if (node->SinceVersion() <= 6) {
+ NodeAttrHelper helper(*node);
+ auto min = helper.Get("min", -3.402e+38f);
+ auto max = helper.Get("max", 3.402e+38f);
+ auto op = graph_ep->GetGraph()->CreateOperation<tim::vx::ops::Clip>(min, max);
+ (*op).BindInputs(inputs).BindOutputs(outputs);
+ graph_ep->GetOps().push_back(std::move(op));
+ } else {
+ switch (inputs[0]->GetDataType()) {
+ case tim::vx::DataType::INT8:
+ ClipImpl<int8_t>(graph_ep, inputs, outputs);
+ break;
+ case tim::vx::DataType::UINT8:
+ ClipImpl<uint8_t>(graph_ep, inputs, outputs);
+ break;
+ case tim::vx::DataType::INT16:
+ ClipImpl<int16_t>(graph_ep, inputs, outputs);
+ break;
+ case tim::vx::DataType::INT32:
+ ClipImpl<int32_t>(graph_ep, inputs, outputs);
+ break;
+ case tim::vx::DataType::FLOAT16:
+ ClipImpl<Ort::Float16_t>(graph_ep, inputs, outputs);
+ break;
+ case tim::vx::DataType::FLOAT32:
+ default:
+ ClipImpl<float>(graph_ep, inputs, outputs);
+ break;
+ }
+ }
+ return true;
+}
+
+} // namespace npu
+
+} // namespace vsi
+} // namespace onnxruntime

onnxruntime/core/providers/vsinpu/builders/impl/clip_op_builder.h

@@ -0,0 +1,59 @@
+/****************************************************************************
+ *
+ * Copyright (c) 2024 Vivante Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ *****************************************************************************/
+#include "core/providers/vsinpu/builders/impl/base_op_builder.h"
+#include "core/providers/shared/utils/utils.h"
+
+namespace onnxruntime {
+namespace vsi {
+namespace npu {
+// template <typename T>
+class ClipOpBuilder final : public BaseOpBuilder {
+ bool IsOpSupported(const onnxruntime::GraphViewer& graph_viewer,
+ const Node* node) const override {
+ if (*node->InputDefs()[0]->Type() == "tensor(int64)") {
+ LOGS_DEFAULT(ERROR) << "Int64 datatype is only used to describe a param in TIM-VX.";
+ return false;
+ }
+ if (node->SinceVersion() > 6) {
+ if (node->InputDefs().size() > 1 && !graph_viewer.IsInitializedTensor(node->InputDefs()[1]->Name())) {
+ LOGS_DEFAULT(ERROR) << "Min/Max value must be const input or attribute.";
+ return false;
+ }
+ }
+ return true;
+ }
+
+ bool HandleBuildOp(vsi::npu::GraphEP* graph_ep,
+ std::vector<std::shared_ptr<tim::vx::Tensor>>& inputs,
+ std::vector<std::shared_ptr<tim::vx::Tensor>>& outputs,
+ const Node* node) override;
+
+ private:
+ template <typename T>
+ struct ClipImpl;
+};
+} // namespace npu
+
+} // namespace vsi
+} // namespace onnxruntime

onnxruntime/core/providers/vsinpu/builders/impl/concat_op_builder.h

@@ -49,8 +49,7 @@ class ConcatOpBuilder : public BaseOpBuilder {
  LOGS_DEFAULT(VERBOSE) << "Creating Concat Op.";
  NodeAttrHelper helper(*node);
  auto axis = helper.Get("axis", 0);
- axis = HandleNegativeAxis(axis, inputs[0]->GetShape().size());
- axis = inputs[0]->GetShape().size() - axis - 1;
+ axis = util::ReverseAxis(axis, inputs[0]->GetShape().size());
  auto op = graph_ep->GetGraph()->CreateOperation<tim::vx::ops::Concat>(static_cast<uint32_t>(axis), inputs.size());
  (*op).BindInputs(inputs).BindOutputs(outputs);
  graph_ep->GetOps().push_back(std::move(op));

onnxruntime/core/providers/vsinpu/builders/impl/flatten_op_builder.h

@@ -40,10 +40,8 @@ class FlattenOpBuilder : public BaseOpBuilder {
  else {
  auto input_shape = inputs[0]->GetShape();
  NodeAttrHelper helper(*node);
- int64_t axis = helper.Get("axis", 0);
- if (axis < 0) {
- axis = HandleNegativeAxis(axis, inputs[0]->GetShape().size()); // handle negative and enforce axis is valid
- }
+ int64_t axis = helper.Get("axis", 1);
+ axis = util::ReverseAxis(static_cast<int32_t>(axis), input_shape.size());
  uint32_t first_dim = 1;
  for (int64_t i = 0; i < axis; i++) {
  first_dim *= inputs[0]->GetShape()[i];

onnxruntime/core/providers/vsinpu/builders/impl/norm_op_builder.h

@@ -0,0 +1,75 @@
+/****************************************************************************
+ *
+ * Copyright (c) 2024 Vivante Corporation
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining a
+ * copy of this software and associated documentation files (the "Software"),
+ * to deal in the Software without restriction, including without limitation
+ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
+ * and/or sell copies of the Software, and to permit persons to whom the
+ * Software is furnished to do so, subject to the following conditions:
+ *
+ * The above copyright notice and this permission notice shall be included in
+ * all copies or substantial portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
+ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ *
+ *****************************************************************************/
+#include "core/providers/vsinpu/builders/impl/base_op_builder.h"
+#include "core/providers/shared/utils/utils.h"
+
+namespace onnxruntime {
+namespace vsi {
+namespace npu {
+enum {
+ input_tensor = 0,
+ scale_tensor = 1,
+ Bias_tensor = 2,
+ mean_tensor = 3,
+ var_tensor = 4
+};
+class BatchNormOpBuilder : public BaseOpBuilder {
+ bool IsOpSupported(const onnxruntime::GraphViewer& graph_viewer,
+ const Node* node) const override {
+ auto input_defs = node->InputDefs();
+ NodeAttrHelper helper(*node);
+ auto training_mode = helper.Get("training_mode", 0);
+ if (training_mode) {
+ LOGS_DEFAULT(WARNING) << "Training is not supported in batch_norm op.";
+ return false;
+ }
+ if (helper.HasAttr("spatial") || node->SinceVersion() < 9) {
+ LOGS_DEFAULT(ERROR) << "VSINPU does not support 'spatial' parameter.";
+ return false;
+ }
+ if (!graph_viewer.IsInitializedTensor(input_defs[scale_tensor]->Name())) {
+ LOGS_DEFAULT(ERROR) << "Not support mean/var/gamma/beta set as dynamic input yet.";
+ return false;
+ }
+
+ return true;
+ }
+ bool HandleBuildOp(vsi::npu::GraphEP* graph_ep,
+ std::vector<std::shared_ptr<tim::vx::Tensor>>& inputs,
+ std::vector<std::shared_ptr<tim::vx::Tensor>>& outputs,
+ const Node* node) override {
+ LOGS_DEFAULT(INFO) << "Creating BatchNorm Op.";
+ NodeAttrHelper helper(*node);
+ auto epsilon = helper.Get("epsilon", 1e-5f);
+ auto op = graph_ep->GetGraph()->CreateOperation<tim::vx::ops::BatchNorm>(epsilon);
+ (*op).BindInput(inputs[input_tensor]).BindInput(inputs[mean_tensor]).BindInput(inputs[var_tensor]).BindInput(inputs[scale_tensor]).BindInput(inputs[Bias_tensor]);
+ (*op).BindOutputs(outputs);
+ graph_ep->GetOps().push_back(std::move(op));
+ return true;
+ }
+};
+} // namespace npu
+
+} // namespace vsi
+} // namespace onnxruntime