diff --git a/include/tvm/relay/qnn/attrs.h b/include/tvm/relay/qnn/attrs.h index 492ed1420672..1cd4c191be51 100644 --- a/include/tvm/relay/qnn/attrs.h +++ b/include/tvm/relay/qnn/attrs.h @@ -187,6 +187,36 @@ struct QnnConv2DAttrs : public tvm::AttrsNode { } }; +/*! \brief Attribute for QNN binary operator */ +struct QnnBinaryOpAttrs : public tvm::AttrsNode { + int32_t lhs_zero_point; + double lhs_scale; + int32_t rhs_zero_point; + double rhs_scale; + int32_t output_zero_point; + double output_scale; + + TVM_DECLARE_ATTRS(QnnBinaryOpAttrs, "relay.attrs.QnnBinaryOpAttrs") { + TVM_ATTR_FIELD(lhs_zero_point) + .describe("The zero_point for the lhs input tensor of this op."); + + TVM_ATTR_FIELD(lhs_scale) + .describe("The scale for the lhs input tensor of this op."); + + TVM_ATTR_FIELD(rhs_zero_point) + .describe("The zero_point for the rhs input tensor of this op."); + + TVM_ATTR_FIELD(rhs_scale) + .describe("The scale for the rhs input tensor of this op."); + + TVM_ATTR_FIELD(output_zero_point) + .describe("The zero_point for the activation of this op."); + + TVM_ATTR_FIELD(output_scale) + .describe("The scale for the activation of this op."); + } +}; + } // namespace qnn } // namespace relay } // namespace tvm diff --git a/python/tvm/relay/qnn/op/qnn.py b/python/tvm/relay/qnn/op/qnn.py index 354e4d24ec9b..570448f60738 100644 --- a/python/tvm/relay/qnn/op/qnn.py +++ b/python/tvm/relay/qnn/op/qnn.py @@ -263,3 +263,45 @@ def conv2d(data, strides, padding, dilation, groups, channels, kernel_size, data_layout, kernel_layout, out_layout, out_dtype) + + +def add(lhs, rhs, lhs_scale, lhs_zero_point, rhs_scale, rhs_zero_point, output_scale, + output_zero_point): + """Quantized addition with numpy-style broadcasting. + + Parameters + ---------- + lhs : relay.Expr + The left hand side quantized input data. + + rhs : relay.Expr + The right hand side quantized input data. + + lhs_scale: float + The scale of the lhs quantized expr. + + lhs_zero_point: int + The zero point of lhs quantized expr. + + rhs_scale: float + The scale of the rhs quantized expr. + + rhs_zero_point: int + The zero point of rhs quantized expr. + + output_scale: float + The scale of the output quantized expr. + + output_zero_point: int + The zero point of output quantized expr. + + Returns + ------- + result : relay.Expr + The computed result. + + """ + return _make.add(lhs, rhs, + lhs_scale, lhs_zero_point, + rhs_scale, rhs_zero_point, + output_scale, output_zero_point) diff --git a/src/relay/qnn/op/add.cc b/src/relay/qnn/op/add.cc new file mode 100644 index 000000000000..4220102f6b38 --- /dev/null +++ b/src/relay/qnn/op/add.cc @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * Copyright (c) 2019 by Contributors + * \file src/relay/qnn/op/add.cc + * \brief QNN add operator. + */ +#include +#include +#include +#include "../../pass/pattern_util.h" +#include "../util.h" +#include "op_common.h" + +namespace tvm { +namespace relay { +namespace qnn { + +/* + * \brief Canonicalizes the QNN add op. + * \param attrs The QNN concatenate attrs. + * \param new_args The new mutated args to the call node. + * \param arg_types The types of input and output. + * \return The sequence of Relay ops for add op. + */ +Expr QnnAddCanonicalize(const Attrs& attrs, const Array& new_args, + const Array& arg_types) { + // Get the attrs. + CHECK_EQ(new_args.size(), 2); + auto& lhs = new_args[0]; + auto& rhs = new_args[1]; + const auto* binary_op_attrs = attrs.as(); + CHECK(binary_op_attrs != nullptr); + auto lhs_scale = binary_op_attrs->lhs_scale; + auto lhs_zero_point = binary_op_attrs->lhs_zero_point; + auto rhs_scale = binary_op_attrs->rhs_scale; + auto rhs_zero_point = binary_op_attrs->rhs_zero_point; + auto output_scale = binary_op_attrs->output_scale; + auto output_zero_point = binary_op_attrs->output_zero_point; + + // Get the input dtype and shape. + CHECK_EQ(arg_types.size(), 3); + auto tensor_type = arg_types[0].as(); + auto input_dtype = tensor_type->dtype; + auto input_shape = tensor_type->shape; + + // FIXME (anijain2305) - The lowering can be further optimized. Instead of inserting requantize in + // the start, we can insert requantize at the end if both input tensors have same qnn params. In + // that case, we can first add the tensors, subtract the zero point, and requantize at the end. + // This can be done in future. + + // Since the input qnn params can be different than output qnn params, we first requantize the + // input tensors to the output qnn params. Then we call relay.add on the requantized inputs. This + // addition results in extra addition of the output zero point. We futher subtract the zero + // point. The whole process can be represented using following equations + // + // scale_c * (Q_c - zp_c) = scale_a * (Q_a - zp_a) + scale_b * (Q_b - zp_b) + // + // After requantizing Q_a and Q_b, equation becomes, + // scale_c * (Q_c - zp_c) = scale_c * (Q_a' - zp_c) + scale_c * (Q_b' - zp_c) + // scale_c * (Q_c - zp_c) = scale_c * (Q_a' + Q_b' - zp_c - zp_c) + // + // Comparing the LHS and RHS, it results in + // Q_c = Q_a' + Q_b' - zp_c + // The add op is done in int32 precision. + + // Requantize LHS if necessary. + auto requantized_lhs = lhs; + if (lhs_scale != output_scale || lhs_zero_point != output_zero_point) { + requantized_lhs = Requantize(lhs, input_shape, lhs_scale, lhs_zero_point, output_scale, + output_zero_point, Int(32)); + } else { + requantized_lhs = Cast(requantized_lhs, Int(32)); + } + + // Requantize RHS if necessary. + auto requantized_rhs = rhs; + if (rhs_scale != output_scale || rhs_zero_point != output_zero_point) { + requantized_rhs = Requantize(rhs, input_shape, rhs_scale, rhs_zero_point, output_scale, + output_zero_point, Int(32)); + } else { + requantized_rhs = Cast(requantized_rhs, Int(32)); + } + + auto output = Add(requantized_lhs, requantized_rhs); + + // Subtract zero point. + if (output_zero_point != 0) { + auto output_zp = MakeConstantScalar(Int(32), output_zero_point); + output = Subtract(output, output_zp); + } + + // Go back to lower precision. + auto q_min = GetQmin(input_dtype); + auto q_max = GetQmax(input_dtype); + output = Clip(output, q_min, q_max); + return Cast(output, input_dtype); +} + +// QNN Addition operator. +QNN_REGISTER_BINARY_OP("add") +.describe("Elementwise add with with broadcasting for quantized tensors.") +.set_support_level(11) +.set_attr("FTVMQnnCanonicalize", QnnAddCanonicalize); + +} // namespace qnn +} // namespace relay +} // namespace tvm diff --git a/src/relay/qnn/op/op_common.h b/src/relay/qnn/op/op_common.h new file mode 100644 index 000000000000..eac18c8c3b98 --- /dev/null +++ b/src/relay/qnn/op/op_common.h @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +/*! + * Copyright (c) 2018 by Contributors + * \file src/relay/qnn/op/op_common.h + * \brief A set of utilities and common functionality for QNN ops. + */ +#ifndef TVM_RELAY_QNN_OP_OP_COMMON_H_ +#define TVM_RELAY_QNN_OP_OP_COMMON_H_ + +#include +#include +#include +#include +#include +#include "../../op/type_relations.h" + +namespace tvm { +namespace relay { +namespace qnn { + +/*! Quick helper macro + * - Expose a positional make function to construct the node. + * - Register op to the registry. + * + * We make the decision to always only expose positional argument. + * We will do rewrapping in the frontend to support language + * sugars such as keyword arguments and default value. + * + * \param OpName the name of registry. + */ +#define QNN_REGISTER_BINARY_OP(OpName) \ + TVM_REGISTER_API("relay.qnn.op._make." OpName) \ + .set_body_typed( \ + [](Expr lhs, Expr rhs, double lhs_scale, int32_t lhs_zero_point, double rhs_scale, \ + int32_t rhs_zero_point, double output_scale, int32_t output_zero_point) { \ + auto attrs = make_node(); \ + attrs->lhs_scale = lhs_scale; \ + attrs->lhs_zero_point = lhs_zero_point; \ + attrs->rhs_scale = rhs_scale; \ + attrs->rhs_zero_point = rhs_zero_point; \ + attrs->output_scale = output_scale; \ + attrs->output_zero_point = output_zero_point; \ + static const Op& op = Op::Get("qnn." OpName); \ + return CallNode::make(op, {lhs, rhs}, Attrs(attrs), {}); \ + }); \ + RELAY_REGISTER_OP("qnn." OpName) \ + .set_num_inputs(2) \ + .add_argument("lhs", "Tensor", "The left hand side quantized tensor.") \ + .add_argument("rhs", "Tensor", "The right hand side quantized tensor.") \ + .add_type_rel("Broadcast", BroadcastRel) + +} // namespace qnn +} // namespace relay +} // namespace tvm + +#endif // TVM_RELAY_QNN_OP_OP_COMMON_H_ diff --git a/tests/python/relay/test_qnn_add.py b/tests/python/relay/test_qnn_add.py new file mode 100644 index 000000000000..e9192064dfc0 --- /dev/null +++ b/tests/python/relay/test_qnn_add.py @@ -0,0 +1,196 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import tvm +import numpy as np +from tvm import relay +from tvm.contrib import graph_runtime +import topi.testing + +def test_tflite_same_io_qnn_params(): + data_dtype = 'uint8' + + x = relay.var("x", shape=(1, 4), dtype=data_dtype) + y = relay.var("y", shape=(1, 4), dtype=data_dtype) + z = relay.qnn.op.add(lhs=x, rhs=y, + lhs_scale=0.00784314, + lhs_zero_point=127, + rhs_scale=0.00784314, + rhs_zero_point=127, + output_scale=0.00784314, + output_zero_point=127) + + func = relay.Function([x, y], z) + mod = relay.Module.from_expr(func) + mod = relay.qnn.transform.CanonicalizeOps()(mod) + func = mod["main"] + + x_datas = [np.array((140, 153, 165, 178)).reshape((1,4)), + np.array((25, 153, 178, 216)).reshape((1,4)), + np.array((25, 153, 216, 165)).reshape((1,4))] + y_datas = [np.array((204, 178, 165, 140)).reshape((1,4)), + np.array((204, 178, 191, 25)).reshape((1,4)), + np.array((204, 178, 25, 191)).reshape((1,4))] + golden_outputs = [np.array((217,204,203,191)).reshape((1, 4)), + np.array((102, 204, 242, 114)).reshape((1,4)), + np.array((102, 204, 114, 229)).reshape((1,4))] + + for i in range(0, 3): + x_data = x_datas[i] + y_data = y_datas[i] + golden_output = golden_outputs[i] + + intrp = relay.create_executor("graph", ctx=tvm.cpu(0), target="llvm") + op_res = intrp.evaluate(func)(x_data, y_data) + np.testing.assert_equal(op_res.asnumpy(), golden_output) + + +def test_tflite_different_io_qnn_params(): + data_dtype = 'uint8' + + x = relay.var("x", shape=(1, 4), dtype=data_dtype) + y = relay.var("y", shape=(1, 4), dtype=data_dtype) + z = relay.qnn.op.add(lhs=x, rhs=y, + lhs_scale=0.0156863, + lhs_zero_point=127, + rhs_scale=0.0117647, + rhs_zero_point=85, + output_scale=0.0235294, + output_zero_point=128) + + func = relay.Function([x, y], z) + mod = relay.Module.from_expr(func) + mod = relay.qnn.transform.CanonicalizeOps()(mod) + func = mod["main"] + + x_datas = [np.array((76, 140, 153, 172)).reshape((1,4)), + np.array((133, 140, 146, 153)).reshape((1,4)), + np.array((76, 140, 172, 146)).reshape((1,4))] + y_datas = [np.array((136, 119, 128, 17)).reshape((1,4)), + np.array((136, 119, 111, 94)).reshape((1,4)), + np.array((136, 119, 17, 128)).reshape((1,4))] + golden_outputs = [np.array((120, 154, 167, 124)).reshape((1, 4)), + np.array((158, 154, 154, 150)).reshape((1,4)), + np.array((120, 154, 124, 163)).reshape((1,4))] + + for i in range(0, 3): + x_data = x_datas[i] + y_data = y_datas[i] + golden_output = golden_outputs[i] + + intrp = relay.create_executor("graph", ctx=tvm.cpu(0), target="llvm") + op_res = intrp.evaluate(func)(x_data, y_data) + np.testing.assert_equal(op_res.asnumpy(), golden_output) + + +def test_saturation(): + # Same params + data_dtype = 'uint8' + x = relay.var("x", shape=(1, 4), dtype=data_dtype) + y = relay.var("y", shape=(1, 4), dtype=data_dtype) + z = relay.qnn.op.add(lhs=x, rhs=y, + lhs_scale=0.125, + lhs_zero_point=0, + rhs_scale=0.125, + rhs_zero_point=0, + output_scale=0.125, + output_zero_point=0) + + func = relay.Function([x, y], z) + mod = relay.Module.from_expr(func) + mod = relay.qnn.transform.CanonicalizeOps()(mod) + func = mod["main"] + + x_data = np.array((255, 1, 1, 0)).reshape((1,4)) + y_data = np.array((255, 255, 128, 0)).reshape((1,4)) + golden_output = np.array((255, 255, 129, 0)).reshape((1, 4)) + + intrp = relay.create_executor("graph", ctx=tvm.cpu(0), target="llvm") + op_res = intrp.evaluate(func)(x_data, y_data) + np.testing.assert_equal(op_res.asnumpy(), golden_output) + + # Same params, different scale + z = relay.qnn.op.add(lhs=x, rhs=y, + lhs_scale=0.125, + lhs_zero_point=0, + rhs_scale=0.125, + rhs_zero_point=0, + output_scale=0.25, + output_zero_point=0) + + func = relay.Function([x, y], z) + mod = relay.Module.from_expr(func) + mod = relay.qnn.transform.CanonicalizeOps()(mod) + func = mod["main"] + + x_data = np.array((255, 1, 1, 0)).reshape((1,4)) + y_data = np.array((255, 255, 127, 0)).reshape((1,4)) + golden_output = np.array((255, 129, 65, 0)).reshape((1, 4)) + + intrp = relay.create_executor("graph", ctx=tvm.cpu(0), target="llvm") + op_res = intrp.evaluate(func)(x_data, y_data) + np.testing.assert_equal(op_res.asnumpy(), golden_output) + + # Same io params, different output scale + z = relay.qnn.op.add(lhs=x, rhs=y, + lhs_scale=0.125, + lhs_zero_point=0, + rhs_scale=0.125, + rhs_zero_point=0, + output_scale=0.25, + output_zero_point=0) + + func = relay.Function([x, y], z) + mod = relay.Module.from_expr(func) + mod = relay.qnn.transform.CanonicalizeOps()(mod) + func = mod["main"] + + x_data = np.array((255, 1, 1, 0)).reshape((1,4)) + y_data = np.array((255, 255, 127, 0)).reshape((1,4)) + golden_output = np.array((255, 129, 65, 0)).reshape((1, 4)) + + intrp = relay.create_executor("graph", ctx=tvm.cpu(0), target="llvm") + op_res = intrp.evaluate(func)(x_data, y_data) + np.testing.assert_equal(op_res.asnumpy(), golden_output) + + # All params different + z = relay.qnn.op.add(lhs=x, rhs=y, + lhs_scale=0.5, + lhs_zero_point=0, + rhs_scale=0.25, + rhs_zero_point=0, + output_scale=0.125, + output_zero_point=0) + + func = relay.Function([x, y], z) + mod = relay.Module.from_expr(func) + mod = relay.qnn.transform.CanonicalizeOps()(mod) + func = mod["main"] + + x_data = np.array((255, 0, 1, 0)).reshape((1,4)) + y_data = np.array((0, 128, 64, 0)).reshape((1,4)) + golden_output = np.array((255, 255, 132, 0)).reshape((1, 4)) + + intrp = relay.create_executor("graph", ctx=tvm.cpu(0), target="llvm") + op_res = intrp.evaluate(func)(x_data, y_data) + np.testing.assert_equal(op_res.asnumpy(), golden_output) + + +if __name__ == '__main__': + test_tflite_same_io_qnn_params() + test_tflite_different_io_qnn_params() + test_saturation()