diff --git a/inference/python_api_test/test_case/infer_test.py b/inference/python_api_test/test_case/infer_test.py index d4f3d88d05..60089d95d0 100644 --- a/inference/python_api_test/test_case/infer_test.py +++ b/inference/python_api_test/test_case/infer_test.py @@ -684,6 +684,9 @@ def trt_bz1_multi_thread_test( use_static=False, use_calib_mode=False, delete_pass_list=None, + dynamic=False, + tuned=False, + shape_range_file="shape_range.pbtxt", ): """ test enable_tensorrt_engine() @@ -711,14 +714,29 @@ def trt_bz1_multi_thread_test( "trt_int8": paddle_infer.PrecisionType.Int8, } self.pd_config.enable_use_gpu(gpu_mem, 0) - self.pd_config.enable_tensorrt_engine( - workspace_size=1 << 30, - max_batch_size=1, - min_subgraph_size=min_subgraph_size, - precision_mode=trt_precision_map[precision], - use_static=use_static, - use_calib_mode=use_calib_mode, - ) + if dynamic: + if tuned: + self.pd_config.collect_shape_range_info("shape_range.pbtxt") + else: + self.pd_config.enable_tensorrt_engine( + workspace_size=1 << 30, + max_batch_size=1, + min_subgraph_size=min_subgraph_size, + precision_mode=trt_precision_map[precision], + use_static=use_static, + use_calib_mode=use_calib_mode, + ) + self.pd_config.enable_tuned_tensorrt_dynamic_shape(shape_range_file, True) + else: + self.pd_config.enable_tensorrt_engine( + workspace_size=1 << 30, + max_batch_size=1, + min_subgraph_size=min_subgraph_size, + precision_mode=trt_precision_map[precision], + use_static=use_static, + use_calib_mode=use_calib_mode, + ) + if delete_pass_list: for ir_pass in delete_pass_list: self.pd_config.delete_pass(ir_pass) diff --git a/inference/python_api_test/test_class_model/test_tnt_small_trt_fp16.py b/inference/python_api_test/test_class_model/test_tnt_small_trt_fp16.py index f1e7777b86..18b78362ad 100644 --- a/inference/python_api_test/test_class_model/test_tnt_small_trt_fp16.py +++ b/inference/python_api_test/test_class_model/test_tnt_small_trt_fp16.py @@ -67,11 +67,19 @@ def test_trt_fp16_more_bz(): del test_suite # destroy class to save memory + # collect shape for trt + test_suite_c = InferenceTest() + test_suite_c.load_config( + model_file=file_path + "/inference.pdmodel", params_file=file_path + "/inference.pdiparams" + ) + test_suite_c.collect_shape_info(model_path=file_path, input_data_dict=input_data_dict, device="gpu") + del test_suite_c # destroy class to save memory + test_suite2 = InferenceTest() test_suite2.load_config( model_file="./TNT_small/inference.pdmodel", params_file="./TNT_small/inference.pdiparams" ) - # TODO:DLTP-58929修复后去掉delete pass + test_suite2.trt_more_bz_test( input_data_dict, output_data_dict, @@ -79,7 +87,8 @@ def test_trt_fp16_more_bz(): max_batch_size=10, min_subgraph_size=30, precision="trt_fp16", - delete_pass_list=["layernorm_shift_partition_fuse_pass", "preln_residual_bias_fuse_pass"], + dynamic=True, + shape_range_file=file_path + "/shape_range.pbtxt", ) del test_suite2 # destroy class to save memory @@ -108,11 +117,19 @@ def test_jetson_trt_fp16_more_bz(): del test_suite # destroy class to save memory + # collect shape for trt + test_suite_c = InferenceTest() + test_suite_c.load_config( + model_file=file_path + "/inference.pdmodel", params_file=file_path + "/inference.pdiparams" + ) + test_suite_c.collect_shape_info(model_path=file_path, input_data_dict=input_data_dict, device="gpu") + del test_suite_c + test_suite2 = InferenceTest() test_suite2.load_config( model_file="./TNT_small/inference.pdmodel", params_file="./TNT_small/inference.pdiparams" ) - # TODO:DLTP-58929修复后去掉delete pass + test_suite2.trt_more_bz_test( input_data_dict, output_data_dict, @@ -120,7 +137,8 @@ def test_jetson_trt_fp16_more_bz(): max_batch_size=10, min_subgraph_size=30, precision="trt_fp16", - delete_pass_list=["layernorm_shift_partition_fuse_pass", "preln_residual_bias_fuse_pass"], + dynamic=True, + shape_range_file=file_path + "/shape_range.pbtxt", ) del test_suite2 # destroy class to save memory @@ -147,15 +165,24 @@ def test_trt_fp16_bz1_multi_thread(): del test_suite # destroy class to save memory + # collect shape for trt + test_suite_c = InferenceTest() + test_suite_c.load_config( + model_file=file_path + "/inference.pdmodel", params_file=file_path + "/inference.pdiparams" + ) + test_suite_c.collect_shape_info(model_path=file_path, input_data_dict=input_data_dict, device="gpu") + del test_suite_c + test_suite2 = InferenceTest() test_suite2.load_config(model_file="./TNT_small/inference.pdmodel", params_file="./TNT_small/inference.pdiparams") - # TODO:DLTP-58929修复后去掉delete pass + test_suite2.trt_bz1_multi_thread_test( input_data_dict, output_data_dict, min_subgraph_size=30, precision="trt_fp16", - delete_pass_list=["layernorm_shift_partition_fuse_pass", "preln_residual_bias_fuse_pass"], + dynamic=True, + shape_range_file="./TNT_small/shape_range.pbtxt", ) del test_suite2 # destroy class to save memory diff --git a/inference/python_api_test/test_class_model/test_tnt_small_trt_fp32.py b/inference/python_api_test/test_class_model/test_tnt_small_trt_fp32.py index 7f493f3706..1deec21e7e 100644 --- a/inference/python_api_test/test_class_model/test_tnt_small_trt_fp32.py +++ b/inference/python_api_test/test_class_model/test_tnt_small_trt_fp32.py @@ -67,18 +67,27 @@ def test_trt_fp32_more_bz(): del test_suite # destroy class to save memory + # collect shape for trt + test_suite_c = InferenceTest() + test_suite_c.load_config( + model_file=file_path + "/inference.pdmodel", params_file=file_path + "/inference.pdiparams" + ) + test_suite_c.collect_shape_info(model_path=file_path, input_data_dict=input_data_dict, device="gpu") + del test_suite_c # destroy class to save memory + test_suite2 = InferenceTest() test_suite2.load_config( model_file="./TNT_small/inference.pdmodel", params_file="./TNT_small/inference.pdiparams" ) - # TODO:DLTP-58929修复后去掉delete pass + test_suite2.trt_more_bz_test( input_data_dict, output_data_dict, max_batch_size=10, min_subgraph_size=30, precision="trt_fp32", - delete_pass_list=["layernorm_shift_partition_fuse_pass", "preln_residual_bias_fuse_pass"], + dynamic=True, + shape_range_file=file_path + "/shape_range.pbtxt", ) del test_suite2 # destroy class to save memory @@ -107,18 +116,27 @@ def test_jetson_trt_fp32_more_bz(): del test_suite # destroy class to save memory + # collect shape for trt + test_suite_c = InferenceTest() + test_suite_c.load_config( + model_file=file_path + "/inference.pdmodel", params_file=file_path + "/inference.pdiparams" + ) + test_suite_c.collect_shape_info(model_path=file_path, input_data_dict=input_data_dict, device="gpu") + del test_suite_c + test_suite2 = InferenceTest() test_suite2.load_config( model_file="./TNT_small/inference.pdmodel", params_file="./TNT_small/inference.pdiparams" ) - # TODO:DLTP-58929修复后去掉delete pass + test_suite2.trt_more_bz_test( input_data_dict, output_data_dict, max_batch_size=10, min_subgraph_size=30, precision="trt_fp32", - delete_pass_list=["layernorm_shift_partition_fuse_pass", "preln_residual_bias_fuse_pass"], + dynamic=True, + shape_range_file=file_path + "/shape_range.pbtxt", ) del test_suite2 # destroy class to save memory @@ -145,15 +163,24 @@ def test_trt_fp32_bz1_multi_thread(): del test_suite # destroy class to save memory + # collect shape for trt + test_suite_c = InferenceTest() + test_suite_c.load_config( + model_file=file_path + "/inference.pdmodel", params_file=file_path + "/inference.pdiparams" + ) + test_suite_c.collect_shape_info(model_path=file_path, input_data_dict=input_data_dict, device="gpu") + del test_suite_c + test_suite2 = InferenceTest() test_suite2.load_config(model_file="./TNT_small/inference.pdmodel", params_file="./TNT_small/inference.pdiparams") - # TODO:DLTP-58929修复后去掉delete pass + test_suite2.trt_bz1_multi_thread_test( input_data_dict, output_data_dict, min_subgraph_size=30, precision="trt_fp32", - delete_pass_list=["layernorm_shift_partition_fuse_pass", "preln_residual_bias_fuse_pass"], + dynamic=True, + shape_range_file=file_path + "/shape_range.pbtxt", ) del test_suite2 # destroy class to save memory