From 7052f8544b89bc70fc5aca4caa242df0e03ed8ff Mon Sep 17 00:00:00 2001 From: zhangjun Date: Thu, 20 Jan 2022 09:27:25 +0000 Subject: [PATCH 01/11] [inference] update convert reduce op&ut,test=develop --- paddle/fluid/inference/tensorrt/op_teller.cc | 16 ++++++++-- .../operators/reduce_ops/reduce_mean_op.cu | 4 ++- .../inference/test_trt_convert_reduce_mean.py | 31 +++++++------------ .../inference/test_trt_convert_reduce_sum.py | 31 +++++++------------ 4 files changed, 39 insertions(+), 43 deletions(-) diff --git a/paddle/fluid/inference/tensorrt/op_teller.cc b/paddle/fluid/inference/tensorrt/op_teller.cc index 878eef016e7d14..3411eb1631db62 100644 --- a/paddle/fluid/inference/tensorrt/op_teller.cc +++ b/paddle/fluid/inference/tensorrt/op_teller.cc @@ -1495,14 +1495,26 @@ bool OpTeller::Tell(const framework::ir::Node* node, bool use_no_calib_int8, return false; } + auto* block = desc.Block(); + if (block == nullptr) { + VLOG(3) << "The block desc is nullptr, we can't continue to analyze. " + "Developers need to check whether block_desc is passed in " + "the pass."; + return false; + } + // The batch size dimension cannot be reduced if it's not dynamic shape. if (!with_dynamic_shape) { if (BOOST_GET_CONST(bool, desc.GetAttr("reduce_all"))) return false; std::vector dim = BOOST_GET_CONST(std::vector, desc.GetAttr("dim")); + auto* x_var_desc = block->FindVar(desc.Input("X")[0]); + const auto input_shape = x_var_desc->GetShape(); + std::cout << "paddle shape: " << input_shape.size() << std::endl; for (auto x : dim) { - if (!x) return false; + if (x == 0 || (x + input_shape.size() == 0)) return false; } + } else { if (BOOST_GET_CONST(bool, desc.GetAttr("reduce_all")) && !BOOST_GET_CONST(bool, desc.GetAttr("keep_dim"))) @@ -1510,7 +1522,7 @@ bool OpTeller::Tell(const framework::ir::Node* node, bool use_no_calib_int8, } if (desc.HasAttr("out_dtype")) { int out_dtype = BOOST_GET_CONST(int32_t, desc.GetAttr("out_dtype")); - if (out_dtype != -1) { + if (!with_dynamic_shape && (out_dtype == 2)) { return false; } } diff --git a/paddle/fluid/operators/reduce_ops/reduce_mean_op.cu b/paddle/fluid/operators/reduce_ops/reduce_mean_op.cu index 197ced2beaac26..30a699e979efc4 100644 --- a/paddle/fluid/operators/reduce_ops/reduce_mean_op.cu +++ b/paddle/fluid/operators/reduce_ops/reduce_mean_op.cu @@ -22,4 +22,6 @@ REGISTER_OP_CUDA_KERNEL( ops::ReduceCudaKernel, ops::ReduceCudaKernel, - ops::ReduceCudaKernel); + ops::ReduceCudaKernel, + ops::ReduceCudaKernel, + ops::ReduceCudaKernel); diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_reduce_mean.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_reduce_mean.py index ba648042dabf75..2ad12be88ac765 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_reduce_mean.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_reduce_mean.py @@ -36,26 +36,27 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: return False if len(attrs[0]["dim"]) == 0: return False - ## skip not use - if attrs[0]["out_dtype"] != -1: - return False return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]]): - return np.random.random([1, 3, 64, 64]).astype(np.float32) + def generate_input1(dtype, attrs: List[Dict[str, Any]]): + if dtype == -1 or dtype == 5: + return np.random.random([1, 3, 64, 64]).astype(np.float32) + elif dtype == 2: + return np.random.random([1, 3, 64, 64]).astype(np.int32) - for keep_dim in [False, True]: + for keep_dim in [True, False]: for dim in [[], [1], [0], [0, 1], [1, 2, 3], [-2, 0, 3], [-3], [-4, 1], [3, 4, 5]]: - for reduce_all in [False, True]: - for out_dtype in [-1, 0, 1]: + for reduce_all in [True, False]: + for out_dtype in [-1, 5]: dics = [{ "keep_dim": keep_dim, "dim": dim, "reduce_all": reduce_all, - "out_dtype": out_dtype + "out_dtype": out_dtype, + "in_dtype": out_dtype, }, {}] ops_config = [{ @@ -75,7 +76,7 @@ def generate_input1(attrs: List[Dict[str, Any]]): weights={}, inputs={ "input_data": TensorConfig(data_gen=partial( - generate_input1, dics)) + generate_input1, out_dtype, dics)) }, outputs=["reduce_output_data"]) @@ -134,16 +135,6 @@ def generate_trt_nodes_num(attrs, dynamic_shape): pass def add_skip_trt_case(self): - def teller1(program_config, predictor_config): - if program_config.ops[0].attrs['out_dtype'] != -1: - return True - return False - - self.add_skip_case( - teller1, SkipReasons.TRT_NOT_IMPLEMENTED, - "NOT Implemented: we will add out_dtype not equal to -1 in the future" - ) - pass def test(self): diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_reduce_sum.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_reduce_sum.py index ba0f61a2768988..68d9a6aa8552cb 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_reduce_sum.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_reduce_sum.py @@ -37,26 +37,27 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: return False if len(attrs[0]["dim"]) == 0: return False - ## skip not use - if attrs[0]["out_dtype"] != -1: - return False return True def sample_program_configs(self): - def generate_input1(attrs: List[Dict[str, Any]]): - return np.random.random([1, 3, 64, 64]).astype(np.float32) + def generate_input1(dtype, attrs: List[Dict[str, Any]]): + if dtype == -1 or dtype == 5: + return np.random.random([1, 3, 64, 64]).astype(np.float32) + elif dtype == 2: + return np.random.random([1, 3, 64, 64]).astype(np.int32) - for keep_dim in [False, True]: + for keep_dim in [True, False]: for dim in [[], [1], [0], [0, 1], [1, 2, 3], [-2, 0, 3], [-3], [-4, 1], [3, 4, 5]]: - for reduce_all in [False, True]: - for out_dtype in [-1, 0, 1]: + for reduce_all in [True, False]: + for out_dtype in [-1, 5]: dics = [{ "keep_dim": keep_dim, "dim": dim, "reduce_all": reduce_all, - "out_dtype": out_dtype + "out_dtype": out_dtype, + "in_dtype": out_dtype, }, {}] ops_config = [{ @@ -76,7 +77,7 @@ def generate_input1(attrs: List[Dict[str, Any]]): weights={}, inputs={ "input_data": TensorConfig(data_gen=partial( - generate_input1, dics)) + generate_input1, out_dtype, dics)) }, outputs=["reduce_output_data"]) @@ -134,16 +135,6 @@ def generate_trt_nodes_num(attrs, dynamic_shape): pass def add_skip_trt_case(self): - def teller1(program_config, predictor_config): - if program_config.ops[0].attrs['out_dtype'] != -1: - return True - return False - - self.add_skip_case( - teller1, SkipReasons.TRT_NOT_IMPLEMENTED, - "NOT Implemented: we will add out_dtype not equal to -1 in the future" - ) - pass def test(self): From 9b7db2bf5f77634e4e84601d2a507ad4b0fe3df8 Mon Sep 17 00:00:00 2001 From: zhangjun Date: Thu, 20 Jan 2022 12:39:03 +0000 Subject: [PATCH 02/11] update --- paddle/fluid/inference/tensorrt/op_teller.cc | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/paddle/fluid/inference/tensorrt/op_teller.cc b/paddle/fluid/inference/tensorrt/op_teller.cc index 3411eb1631db62..c6674480f0b194 100644 --- a/paddle/fluid/inference/tensorrt/op_teller.cc +++ b/paddle/fluid/inference/tensorrt/op_teller.cc @@ -1490,8 +1490,6 @@ bool OpTeller::Tell(const framework::ir::Node* node, bool use_no_calib_int8, VLOG(3) << "the " << op_type << " does not have attr (keep_dim or dim or " "reduce_all)"; - std::cout << "attr " << desc.HasAttr("keep_dim") << " " - << desc.HasAttr("dim") << " " << desc.HasAttr("reduce_all"); return false; } @@ -1510,7 +1508,6 @@ bool OpTeller::Tell(const framework::ir::Node* node, bool use_no_calib_int8, BOOST_GET_CONST(std::vector, desc.GetAttr("dim")); auto* x_var_desc = block->FindVar(desc.Input("X")[0]); const auto input_shape = x_var_desc->GetShape(); - std::cout << "paddle shape: " << input_shape.size() << std::endl; for (auto x : dim) { if (x == 0 || (x + input_shape.size() == 0)) return false; } @@ -1522,7 +1519,9 @@ bool OpTeller::Tell(const framework::ir::Node* node, bool use_no_calib_int8, } if (desc.HasAttr("out_dtype")) { int out_dtype = BOOST_GET_CONST(int32_t, desc.GetAttr("out_dtype")); - if (!with_dynamic_shape && (out_dtype == 2)) { + if (!with_dynamic_shape && + (static_cast(out_dtype) == + framework::proto::VarType::INT32)) { return false; } } From 5ca9ee9c96685dbc3d6ae97e2a210dda39451bb5 Mon Sep 17 00:00:00 2001 From: zhangjun Date: Thu, 20 Jan 2022 15:34:22 +0000 Subject: [PATCH 03/11] update --- paddle/fluid/inference/tensorrt/op_teller.cc | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/paddle/fluid/inference/tensorrt/op_teller.cc b/paddle/fluid/inference/tensorrt/op_teller.cc index 3411eb1631db62..d220df974533bf 100644 --- a/paddle/fluid/inference/tensorrt/op_teller.cc +++ b/paddle/fluid/inference/tensorrt/op_teller.cc @@ -1504,11 +1504,11 @@ bool OpTeller::Tell(const framework::ir::Node* node, bool use_no_calib_int8, } // The batch size dimension cannot be reduced if it's not dynamic shape. + auto* x_var_desc = block->FindVar(desc.Input("X")[0]); if (!with_dynamic_shape) { if (BOOST_GET_CONST(bool, desc.GetAttr("reduce_all"))) return false; std::vector dim = BOOST_GET_CONST(std::vector, desc.GetAttr("dim")); - auto* x_var_desc = block->FindVar(desc.Input("X")[0]); const auto input_shape = x_var_desc->GetShape(); std::cout << "paddle shape: " << input_shape.size() << std::endl; for (auto x : dim) { @@ -1520,11 +1520,17 @@ bool OpTeller::Tell(const framework::ir::Node* node, bool use_no_calib_int8, !BOOST_GET_CONST(bool, desc.GetAttr("keep_dim"))) return false; } - if (desc.HasAttr("out_dtype")) { - int out_dtype = BOOST_GET_CONST(int32_t, desc.GetAttr("out_dtype")); - if (!with_dynamic_shape && (out_dtype == 2)) { - return false; - } + + auto dtype = x_var_desc->GetDataType(); + if (dtype != + framework::proto::VarType::INT32 && dtype != framework::proto::VarType::FP32) { + VLOG(3) << "reduce op input data type must be int32 or float32"; + return false; + } + + // int32 not support if it's not dynamic shape. + if (!with_dynamic_shape && (dtype == framework::proto::VarType::INT32)) { + return false; } } #if IS_TRT_VERSION_GE(7000) From fec3c5b97a551c66f471ea481e495e5186b19568 Mon Sep 17 00:00:00 2001 From: zhangjun Date: Thu, 20 Jan 2022 15:45:23 +0000 Subject: [PATCH 04/11] update --- paddle/fluid/inference/tensorrt/op_teller.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/fluid/inference/tensorrt/op_teller.cc b/paddle/fluid/inference/tensorrt/op_teller.cc index ffd2f579b64297..fcb48906905245 100644 --- a/paddle/fluid/inference/tensorrt/op_teller.cc +++ b/paddle/fluid/inference/tensorrt/op_teller.cc @@ -1519,8 +1519,8 @@ bool OpTeller::Tell(const framework::ir::Node* node, bool use_no_calib_int8, } auto dtype = x_var_desc->GetDataType(); - if (dtype != - framework::proto::VarType::INT32 && dtype != framework::proto::VarType::FP32) { + if (dtype != framework::proto::VarType::INT32 && + dtype != framework::proto::VarType::FP32) { VLOG(3) << "reduce op input data type must be int32 or float32"; return false; } From 132a0eb5f4aefe3a2754f613c429ec6613ab5f31 Mon Sep 17 00:00:00 2001 From: zhangjun Date: Fri, 21 Jan 2022 08:36:38 +0000 Subject: [PATCH 05/11] add int32 support --- paddle/fluid/inference/tensorrt/convert/reduce_op.cc | 1 + .../unittests/ir/inference/test_trt_convert_reduce_mean.py | 2 +- .../tests/unittests/ir/inference/test_trt_convert_reduce_sum.py | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/paddle/fluid/inference/tensorrt/convert/reduce_op.cc b/paddle/fluid/inference/tensorrt/convert/reduce_op.cc index f3c4059b8e6456..c076dd02192c00 100644 --- a/paddle/fluid/inference/tensorrt/convert/reduce_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/reduce_op.cc @@ -83,6 +83,7 @@ class ReduceOpConverter : public OpConverter { } auto output_name = op_desc.Output("Out")[0]; + layer->getOutput(0)->setType(layer->getInput(0)->getType()); RreplenishLayerAndOutput(layer, op_type, {output_name}, test_mode); } diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_reduce_mean.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_reduce_mean.py index 2ad12be88ac765..29aef6809ea020 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_reduce_mean.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_reduce_mean.py @@ -50,7 +50,7 @@ def generate_input1(dtype, attrs: List[Dict[str, Any]]): for dim in [[], [1], [0], [0, 1], [1, 2, 3], [-2, 0, 3], [-3], [-4, 1], [3, 4, 5]]: for reduce_all in [True, False]: - for out_dtype in [-1, 5]: + for out_dtype in [-1, 2, 5]: dics = [{ "keep_dim": keep_dim, "dim": dim, diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_reduce_sum.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_reduce_sum.py index 68d9a6aa8552cb..2a7e673d4203a9 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_reduce_sum.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_reduce_sum.py @@ -51,7 +51,7 @@ def generate_input1(dtype, attrs: List[Dict[str, Any]]): for dim in [[], [1], [0], [0, 1], [1, 2, 3], [-2, 0, 3], [-3], [-4, 1], [3, 4, 5]]: for reduce_all in [True, False]: - for out_dtype in [-1, 5]: + for out_dtype in [-1, 2, 5]: dics = [{ "keep_dim": keep_dim, "dim": dim, From 82bfef7d143d92f08d75c49ff66e359b08754dd0 Mon Sep 17 00:00:00 2001 From: zhangjun Date: Fri, 21 Jan 2022 08:40:05 +0000 Subject: [PATCH 06/11] add int32 support --- paddle/fluid/inference/tensorrt/op_teller.cc | 5 ----- 1 file changed, 5 deletions(-) diff --git a/paddle/fluid/inference/tensorrt/op_teller.cc b/paddle/fluid/inference/tensorrt/op_teller.cc index fcb48906905245..5919938ad75788 100644 --- a/paddle/fluid/inference/tensorrt/op_teller.cc +++ b/paddle/fluid/inference/tensorrt/op_teller.cc @@ -1524,11 +1524,6 @@ bool OpTeller::Tell(const framework::ir::Node* node, bool use_no_calib_int8, VLOG(3) << "reduce op input data type must be int32 or float32"; return false; } - - // int32 not support if it's not dynamic shape. - if (!with_dynamic_shape && (dtype == framework::proto::VarType::INT32)) { - return false; - } } #if IS_TRT_VERSION_GE(7000) if (op_type == "tile") { From eaf47d16bcd779aeeaa927cc2f512eed223cdf03 Mon Sep 17 00:00:00 2001 From: zhangjun Date: Fri, 21 Jan 2022 09:03:33 +0000 Subject: [PATCH 07/11] add comments --- paddle/fluid/inference/tensorrt/convert/reduce_op.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/paddle/fluid/inference/tensorrt/convert/reduce_op.cc b/paddle/fluid/inference/tensorrt/convert/reduce_op.cc index c076dd02192c00..7c5eaa309ef18a 100644 --- a/paddle/fluid/inference/tensorrt/convert/reduce_op.cc +++ b/paddle/fluid/inference/tensorrt/convert/reduce_op.cc @@ -83,6 +83,7 @@ class ReduceOpConverter : public OpConverter { } auto output_name = op_desc.Output("Out")[0]; + // Ensure that the output type and input type are consistent. layer->getOutput(0)->setType(layer->getInput(0)->getType()); RreplenishLayerAndOutput(layer, op_type, {output_name}, test_mode); } From f0fa5b8bc4ca4f36c2b8337ac6b4aea0ff292b96 Mon Sep 17 00:00:00 2001 From: zhangjun Date: Mon, 24 Jan 2022 02:49:25 +0000 Subject: [PATCH 08/11] trt < 7.0 do not support int32 --- paddle/fluid/inference/tensorrt/op_teller.cc | 7 +++++++ .../unittests/ir/inference/test_trt_convert_reduce_mean.py | 5 +++++ 2 files changed, 12 insertions(+) diff --git a/paddle/fluid/inference/tensorrt/op_teller.cc b/paddle/fluid/inference/tensorrt/op_teller.cc index 5919938ad75788..09d5663652d835 100644 --- a/paddle/fluid/inference/tensorrt/op_teller.cc +++ b/paddle/fluid/inference/tensorrt/op_teller.cc @@ -1519,11 +1519,18 @@ bool OpTeller::Tell(const framework::ir::Node* node, bool use_no_calib_int8, } auto dtype = x_var_desc->GetDataType(); +#if IS_TRT_VERSION_GE(7000) if (dtype != framework::proto::VarType::INT32 && dtype != framework::proto::VarType::FP32) { VLOG(3) << "reduce op input data type must be int32 or float32"; return false; } +#else + if (dtype != framework::proto::VarType::FP32) { + VLOG(3) << "reduce op input data type must be float32 using TensorRT < 7.0"; + return false; + } +#endif } #if IS_TRT_VERSION_GE(7000) if (op_type == "tile") { diff --git a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_reduce_mean.py b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_reduce_mean.py index 29aef6809ea020..2e1e04870b926b 100644 --- a/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_reduce_mean.py +++ b/python/paddle/fluid/tests/unittests/ir/inference/test_trt_convert_reduce_mean.py @@ -37,6 +37,11 @@ def is_program_valid(self, program_config: ProgramConfig) -> bool: if len(attrs[0]["dim"]) == 0: return False + ver = paddle_infer.get_trt_compile_version() + if ver[0] * 1000 + ver[1] * 100 + ver[0] * 10 < 7000: + if attrs[0]['out_dtype'] == 2: + return False + return True def sample_program_configs(self): From 2dbbe96a53c971c1b59223b7a404830eb93ea13e Mon Sep 17 00:00:00 2001 From: zhangjun Date: Mon, 24 Jan 2022 04:51:05 +0000 Subject: [PATCH 09/11] test=develop From 538cb0109cf9ae1ed26ffbe998f7d9dad6e2f1ab Mon Sep 17 00:00:00 2001 From: zhangjun Date: Mon, 24 Jan 2022 14:30:52 +0000 Subject: [PATCH 10/11] update --- paddle/fluid/inference/tensorrt/op_teller.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/paddle/fluid/inference/tensorrt/op_teller.cc b/paddle/fluid/inference/tensorrt/op_teller.cc index 09d5663652d835..3bda0d7ed1787b 100644 --- a/paddle/fluid/inference/tensorrt/op_teller.cc +++ b/paddle/fluid/inference/tensorrt/op_teller.cc @@ -1527,7 +1527,8 @@ bool OpTeller::Tell(const framework::ir::Node* node, bool use_no_calib_int8, } #else if (dtype != framework::proto::VarType::FP32) { - VLOG(3) << "reduce op input data type must be float32 using TensorRT < 7.0"; + VLOG(3) + << "reduce op input data type must be float32 using TensorRT < 7.0"; return false; } #endif From d8b47c74b6a0e3e0f900476da5c5554956ad3abc Mon Sep 17 00:00:00 2001 From: zhangjun Date: Mon, 24 Jan 2022 14:33:41 +0000 Subject: [PATCH 11/11] test=develop