[QNN EP] Update to QNN SDK 2.24.0 (#21463)

### Description
- Update pipelines to use QNN SDK 2.24 by default
- Update QNN_Nuget_Windows pipeline to build csharp solution without
mobile projects (fixes errors).
- Implement workaround for QNN 2.24 validation bug for LayerNorm ops
without an explicit bias input.
- Enable Relu unit test, which now passes due to the fact Relu is no
longer fused into QuantizeLinear for QNN EP.
- Fix bug where a negative quantization axis is not properly normalized
for per-channel int4 conv.



### Motivation and Context
Update QNN SDk.

onnxruntime/core/providers/qnn/builder/opbuilder/layer_norm_op_builder.cc

@@ -1,9 +1,11 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
+#include <cassert>
 #include "core/providers/common.h"
 #include "core/providers/shared/utils/utils.h"
 #include "core/framework/tensorprotoutils.h"
+#include "core/providers/qnn/builder/qnn_utils.h"
 #include "core/providers/qnn/builder/qnn_model_wrapper.h"
 #include "core/providers/qnn/builder/op_builder_factory.h"
 #include "core/common/safeint.h"
@@ -24,6 +26,11 @@ class LayerNormOpBuilder : public BaseOpBuilder {
  const logging::Logger& logger) const override final ORT_MUST_USE_RESULT;
 
  protected:
+ Status ProcessInputs(QnnModelWrapper& qnn_model_wrapper,
+ const NodeUnit& node_unit,
+ const logging::Logger& logger,
+ std::vector<std::string>& input_names,
+ bool do_op_validation) const override ORT_MUST_USE_RESULT;
  Status ProcessAttributesAndOutputs(QnnModelWrapper& qnn_model_wrapper,
  const NodeUnit& node_unit,
  std::vector<std::string>&& input_names,
@@ -55,6 +62,91 @@ Status LayerNormOpBuilder::IsOpSupported(QnnModelWrapper& qnn_model_wrapper,
  return AddToModelBuilder(qnn_model_wrapper, node_unit, logger, true);
 }
 
+Status LayerNormOpBuilder::ProcessInputs(QnnModelWrapper& qnn_model_wrapper,
+ const NodeUnit& node_unit,
+ const logging::Logger& logger,
+ std::vector<std::string>& input_names,
+ bool do_op_validation) const {
+ ORT_UNUSED_PARAMETER(do_op_validation);
+
+ const auto& inputs = node_unit.Inputs();
+ const auto input_count = inputs.size();
+ constexpr size_t X_IDX = 0;
+ constexpr size_t SCALE_IDX = 1;
+ constexpr size_t BIAS_IDX = 2;
+
+ // Input[0] (X, required)
+ ORT_RETURN_IF_ERROR(ProcessInput(qnn_model_wrapper, inputs[X_IDX], logger, input_names));
+
+ // Input[1] (scale, required)
+ ORT_RETURN_IF_ERROR(ProcessInput(qnn_model_wrapper, inputs[SCALE_IDX], logger, input_names));
+
+ // Input[2] (bias, optional)
+ const bool has_bias_input = input_count > BIAS_IDX && inputs[BIAS_IDX].node_arg.Exists();
+ if (has_bias_input) {
+ ORT_RETURN_IF_ERROR(ProcessInput(qnn_model_wrapper, inputs[BIAS_IDX], logger, input_names));
+ }
+
+#if QNN_API_VERSION_MAJOR == 2 && QNN_API_VERSION_MINOR == 17
+ if (!has_bias_input && IsNpuBackend(qnn_model_wrapper.GetQnnBackendType())) {
+ // Bias is implicit. QNN SDK 2.24 (QNN API version 2.17) has a validation bug for implicit bias inputs, so provide
+ // an explicit bias of all 0 (quantized int32).
+ TensorInfo x_input_info = {};
+ ORT_RETURN_IF_ERROR(qnn_model_wrapper.GetTensorInfo(inputs[X_IDX], x_input_info));
+
+ TensorInfo scale_input_info = {};
+ ORT_RETURN_IF_ERROR(qnn_model_wrapper.GetTensorInfo(inputs[SCALE_IDX], scale_input_info));
+
+ if (x_input_info.quant_param.IsPerTensor(/*include_bw*/ true) && scale_input_info.quant_param.IsQuantized()) {
+ const std::string bias_name = qnn::utils::GetNodeName(node_unit) + "_implicit_bias_ort_qnn_ep";
+
+ // Make dummy bias input have the same shape as the scale input.
+ std::vector<uint32_t> bias_shape = scale_input_info.shape;
+ size_t num_bias_elems = 1;
+ for (size_t i = 0; i < bias_shape.size(); i++) {
+ num_bias_elems *= static_cast<size_t>(bias_shape[i]);
+ }
+
+ // Bias static input should be all zeros.
+ std::vector<uint8_t> bias_bytes(num_bias_elems * sizeof(int32_t), 0);
+
+ // Bias's quantization scale should be the product of the other inputs' quantization scales.
+ std::vector<float> input0_quant_scales;
+ std::vector<float> input1_quant_scales;
+ ORT_RETURN_IF_ERROR(x_input_info.quant_param.GetScales(input0_quant_scales));
+ ORT_RETURN_IF_ERROR(scale_input_info.quant_param.GetScales(input1_quant_scales));
+
+ const size_t num_bias_scales_offsets = input1_quant_scales.size();
+ assert(input0_quant_scales.size() == 1); // Expected for per-tensor.
+ ORT_RETURN_IF_NOT(num_bias_scales_offsets >= input0_quant_scales.size(),
+ "Input[1] should have >= 1 quantization scale values");
+
+ std::vector<float> bias_scales(num_bias_scales_offsets);
+ for (size_t i = 0; i < num_bias_scales_offsets; i++) {
+ bias_scales[i] = input0_quant_scales[0] * input1_quant_scales[i];
+ }
+
+ std::vector<int32_t> bias_offsets(num_bias_scales_offsets, 0); // Bias's zero-points should be all zeros.
+ QnnQuantParamsWrapper bias_qparams;
+
+ if (scale_input_info.quant_param.IsPerChannel()) {
+ bias_qparams = QnnQuantParamsWrapper(bias_scales, bias_offsets, /*axis*/ 0, /*is_int4*/ false);
+ } else {
+ bias_qparams = QnnQuantParamsWrapper(bias_scales[0], bias_offsets[0]);
+ }
+
+ auto tensor_wrapper = QnnTensorWrapper(bias_name, QNN_TENSOR_TYPE_STATIC, QNN_DATATYPE_SFIXED_POINT_32,
+ std::move(bias_qparams), std::move(bias_shape), std::move(bias_bytes));
+
+ qnn_model_wrapper.AddTensorWrapper(std::move(tensor_wrapper));
+ input_names.push_back(bias_name);
+ }
+ }
+#endif
+
+ return Status::OK();
+}
+
 Status LayerNormOpBuilder::ProcessAttributesAndOutputs(QnnModelWrapper& qnn_model_wrapper,
  const NodeUnit& node_unit,
  std::vector<std::string>&& input_names,

onnxruntime/core/providers/qnn/builder/qnn_model_wrapper.cc

@@ -442,6 +442,16 @@ Status QnnModelWrapper::IsPerChannelQuantized(const onnxruntime::NodeUnitIODef&
 
  if (is_per_channel) {
  axis = io_def.quant_param->axis.value_or(1); // 1 is default axis for Q/DQ ops.
+ if (axis < 0) {
+ // Normalize negative axis by adding rank.
+ const auto* tensor_shape_proto = io_def.node_arg.Shape();
+ ORT_RETURN_IF_NOT(tensor_shape_proto != nullptr, "NULL tensor shape proto");
+
+ const int rank = tensor_shape_proto->dim_size();
+ ORT_RETURN_IF_NOT(rank > 0, "Per-channel quantized tensor should be of rank > 0");
+
+ axis += rank;
+ }
  }
 
  return Status::OK();

onnxruntime/core/providers/qnn/builder/qnn_quant_params_wrapper.cc

@@ -30,13 +30,118 @@ QnnQuantParamsWrapper& QnnQuantParamsWrapper::operator=(const QnnQuantParamsWrap
  return *this;
 }
 
+// Construct per-tensor quantization params.
 QnnQuantParamsWrapper::QnnQuantParamsWrapper(float scale, int32_t offset) {
  params_.encodingDefinition = QNN_DEFINITION_DEFINED;
  params_.quantizationEncoding = QNN_QUANTIZATION_ENCODING_SCALE_OFFSET;
  params_.scaleOffsetEncoding.scale = scale;
  params_.scaleOffsetEncoding.offset = offset;
 }
 
+// Construct a per-channel quantization param.
+QnnQuantParamsWrapper::QnnQuantParamsWrapper(gsl::span<const float> scales, gsl::span<const int32_t> offsets,
+ int32_t axis, bool is_int4) {
+ assert(scales.size() == offsets.size()); // Logic error if sizes don't match.
+ const uint32_t num_elems = static_cast<uint32_t>(scales.size());
+ params_.encodingDefinition = QNN_DEFINITION_DEFINED;
+
+ if (is_int4) {
+ params_.quantizationEncoding = QNN_QUANTIZATION_ENCODING_BW_AXIS_SCALE_OFFSET;
+ params_.bwAxisScaleOffsetEncoding.numElements = num_elems;
+ params_.bwAxisScaleOffsetEncoding.axis = axis;
+ params_.bwAxisScaleOffsetEncoding.bitwidth = 4;
+
+ // Deep copy to the scales[] and offsets[] arrays
+ if (num_elems > 0) {
+ const size_t num_scale_bytes = num_elems * sizeof(float);
+ const size_t num_zp_bytes = num_elems * sizeof(int32_t);
+ const size_t num_bytes = num_scale_bytes + num_zp_bytes;
+ constexpr std::uintptr_t align = alignof(float);
+ static_assert(alignof(float) == alignof(int32_t));
+
+ per_channel_data_ = std::make_unique<char[]>(num_bytes + align);
+ char* scales_begin = ALIGN_PTR_UP(per_channel_data_.get(), align, char*);
+ char* zps_begin = scales_begin + num_scale_bytes;
+
+ std::memcpy(scales_begin, scales.data(), num_scale_bytes);
+ std::memcpy(zps_begin, offsets.data(), num_zp_bytes);
+ params_.bwAxisScaleOffsetEncoding.scales = reinterpret_cast<float*>(scales_begin);
+ params_.bwAxisScaleOffsetEncoding.offsets = reinterpret_cast<int32_t*>(zps_begin);
+ } else {
+ params_.bwAxisScaleOffsetEncoding.scales = nullptr;
+ params_.bwAxisScaleOffsetEncoding.offsets = nullptr;
+ }
+ } else {
+ params_.quantizationEncoding = QNN_QUANTIZATION_ENCODING_AXIS_SCALE_OFFSET;
+ params_.axisScaleOffsetEncoding.numScaleOffsets = num_elems;
+ params_.axisScaleOffsetEncoding.axis = axis;
+
+ // Deep copy to the scaleOffset data.
+ if (num_elems > 0) {
+ const size_t num_bytes = num_elems * sizeof(Qnn_ScaleOffset_t);
+ constexpr std::uintptr_t align = alignof(Qnn_ScaleOffset_t);
+ per_channel_data_ = std::make_unique<char[]>(num_bytes + align);
+ Qnn_ScaleOffset_t* aligned_dst = ALIGN_PTR_UP(per_channel_data_.get(), align, Qnn_ScaleOffset_t*);
+
+ for (size_t i = 0; i < static_cast<uint32_t>(num_elems); i++) {
+ aligned_dst[i].offset = offsets[i];
+ aligned_dst[i].scale = scales[i];
+ }
+
+ params_.axisScaleOffsetEncoding.scaleOffset = aligned_dst;
+ } else {
+ params_.axisScaleOffsetEncoding.scaleOffset = nullptr;
+ }
+ }
+}
+
+// Get a copy of scales. Works for both per-tensor and per-channel.
+Status QnnQuantParamsWrapper::GetScales(/*out*/ std::vector<float>& scales) const {
+ ORT_RETURN_IF_NOT(params_.encodingDefinition == QNN_DEFINITION_DEFINED, "Unquantized qparams does not have scales");
+
+ switch (params_.quantizationEncoding) {
+ case QNN_QUANTIZATION_ENCODING_SCALE_OFFSET:
+ scales.resize(1);
+ scales[0] = params_.scaleOffsetEncoding.scale;
+ break;
+ case QNN_QUANTIZATION_ENCODING_BW_SCALE_OFFSET:
+ scales.resize(1);
+ scales[0] = params_.bwScaleOffsetEncoding.scale;
+ break;
+ case QNN_QUANTIZATION_ENCODING_AXIS_SCALE_OFFSET: {
+ const uint32_t num_elems = params_.axisScaleOffsetEncoding.numScaleOffsets;
+ scales.resize(num_elems);
+
+ if (num_elems > 0) {
+ gsl::span<const Qnn_ScaleOffset_t> scale_offsets(params_.axisScaleOffsetEncoding.scaleOffset, num_elems);
+
+ for (size_t i = 0; i < num_elems; i++) {
+ scales[i] = scale_offsets[i].scale;
+ }
+ }
+ break;
+ }
+ case QNN_QUANTIZATION_ENCODING_BW_AXIS_SCALE_OFFSET: {
+ const uint32_t num_elems = params_.bwAxisScaleOffsetEncoding.numElements;
+ scales.resize(num_elems);
+
+ // Deep copy the scales[] and offsets[] arrays
+ if (num_elems > 0) {
+ gsl::span<const float> src_scales(params_.bwAxisScaleOffsetEncoding.scales, num_elems);
+ for (size_t i = 0; i < num_elems; i++) {
+ scales[i] = src_scales[i];
+ }
+ }
+ break;
+ }
+ default:
+ return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Unsupported QNN quantization encoding: ",
+ params_.quantizationEncoding);
+ }
+
+ return Status::OK();
+}
+
 QnnQuantParamsWrapper QnnQuantParamsWrapper::Copy() const {
  return QnnQuantParamsWrapper(*this);
 }
@@ -199,7 +304,7 @@ Status QnnQuantParamsWrapper::Init(const QnnModelWrapper& qnn_model_wrapper, con
 
  params_.encodingDefinition = QNN_DEFINITION_DEFINED;
  params_.quantizationEncoding = QNN_QUANTIZATION_ENCODING_BW_AXIS_SCALE_OFFSET;
- params_.bwAxisScaleOffsetEncoding.axis = static_cast<int32_t>(*(ort_quant_params->axis));
+ params_.bwAxisScaleOffsetEncoding.axis = static_cast<int32_t>(axis);
  params_.bwAxisScaleOffsetEncoding.bitwidth = 4;
  params_.bwAxisScaleOffsetEncoding.numElements = static_cast<uint32_t>(num_elems);
 

onnxruntime/core/providers/qnn/builder/qnn_quant_params_wrapper.h

@@ -3,6 +3,7 @@
 
 #pragma once
 #include <memory>
+#include <vector>
 #include "QnnTypes.h"
 #include "core/common/common.h"
 #include <gsl/gsl>
@@ -26,6 +27,9 @@ class QnnQuantParamsWrapper {
  // Construct a per-tensor quantization param (SCALE_OFFSET)
  QnnQuantParamsWrapper(float scale, int32_t offset);
 
+ // Construct a per-channel quantization param.
+ QnnQuantParamsWrapper(gsl::span<const float> scales, gsl::span<const int32_t> offsets, int32_t axis, bool is_int4);
+
  Qnn_QuantizeParams_t& Get() { return params_; }
  const Qnn_QuantizeParams_t& Get() const { return params_; }
 
@@ -54,6 +58,9 @@ class QnnQuantParamsWrapper {
  (params_.quantizationEncoding == QNN_QUANTIZATION_ENCODING_BW_AXIS_SCALE_OFFSET));
  }
 
+ // Get a copy of scales. Works for both per-tensor and per-channel.
+ Status GetScales(/*out*/ std::vector<float>& scales) const;
+
  // Handle transposing of a per-channel quantized tensor. The quantization parameter's axis
  // must be transposed using the inverse permutation of the Transpose.
  template <typename IntType>

onnxruntime/test/providers/qnn/conv_test.cc

@@ -178,18 +178,22 @@ static GetTestQDQModelFn<ActivationQType> BuildQDQPerChannelConvTestCase(const s
  ORT_ENFORCE(weights_def.IsInitializer() && weights_def.IsRawData());
  std::vector<float> weight_scales;
  std::vector<WeightQType> weight_zero_points;
+ TensorShape weights_shape = weights_def.GetTensorShape();
+ int64_t pos_weight_quant_axis = weight_quant_axis;
+ if (pos_weight_quant_axis < 0) {
+ pos_weight_quant_axis += static_cast<int64_t>(weights_shape.NumDimensions());
+ }
  GetTestInputQuantParamsPerChannel<WeightQType>(weights_def, weight_scales, weight_zero_points,
- static_cast<size_t>(weight_quant_axis), true);
+ static_cast<size_t>(pos_weight_quant_axis), true);
 
- TensorShape weights_shape = weights_def.GetTensorShape();
  std::vector<WeightQType> quantized_weights;
  size_t num_weight_storage_elems = weights_shape.Size();
  if constexpr (std::is_same_v<WeightQType, Int4x2> || std::is_same_v<WeightQType, UInt4x2>) {
  num_weight_storage_elems = Int4x2::CalcNumInt4Pairs(weights_shape.Size());
  }
  quantized_weights.resize(num_weight_storage_elems);
  QuantizeValues<float, WeightQType>(weights_def.GetRawData(), quantized_weights, weights_shape,
- weight_scales, weight_zero_points, weight_quant_axis);
+ weight_scales, weight_zero_points, pos_weight_quant_axis);
 
  NodeArg* weights_initializer = builder.MakeInitializer<WeightQType>(weights_def.GetShape(), quantized_weights);
  NodeArg* weights_dq = builder.MakeIntermediate();
@@ -760,6 +764,34 @@ TEST_F(QnnHTPBackendTests, ConvU16S4S32_PerChannel) {
  21); // opset
 }
 
+// Test per-channel QDQ Conv with INT4 weights and a negative weight quantization axis that still points to dimension 0.
+TEST_F(QnnHTPBackendTests, ConvU16S4S32_PerChannel_NegativeWeightQuantAxis) {
+ std::vector<int64_t> input_shape = {1, 2, 4, 4};
+ std::vector<int64_t> weight_shape = {3, 2, 2, 2};
+ std::vector<int64_t> bias_shape = {3};
+
+ TestInputDef<float> input_def(input_shape, false,
+ GetFloatDataInRange(0.0f, 1.0f, TensorShape(input_shape).Size()));
+ TestInputDef<float> weight_def(weight_shape, true,
+ GetFloatDataInRange(-1.0f, 5.0f, TensorShape(weight_shape).Size()));
+ TestInputDef<float> bias_def(bias_shape, true,
+ GetFloatDataInRange(-1.0f, 1.0f, TensorShape(bias_shape).Size()));
+
+ RunHTPConvOpPerChannelTest<uint8_t, Int4x2>("Conv",
+ input_def,
+ weight_def,
+ bias_def,
+ -4, // negative weight quant axis (same as 0)
+ {1, 1}, // Strides
+ {0, 0, 0, 0}, // Pads
+ {1, 1}, // Dilations
+ 1, // default group
+ "NOTSET",
+ ExpectedEPNodeAssignment::All,
+ false, // use_qdq_contrib_ops
+ 21); // opset
+}
+
 // Test per-channel QDQ Conv with INT4 weights. in0: u16, in1 (weight): s4, in2 (bias): s32, out: u8
 // TODO(adrianlizarraga): Investigate inaccuracy for QNN EP.
 //