diff --git a/paddle/fluid/eager/auto_code_generator/generator/CMakeLists.txt b/paddle/fluid/eager/auto_code_generator/generator/CMakeLists.txt index 70e13ee3f38ef9..6416af0218f430 100644 --- a/paddle/fluid/eager/auto_code_generator/generator/CMakeLists.txt +++ b/paddle/fluid/eager/auto_code_generator/generator/CMakeLists.txt @@ -44,6 +44,9 @@ set(nodes_h_path # StringTensor only needs forward api set(fwd_api_yaml_path "${PADDLE_SOURCE_DIR}/paddle/phi/ops/yaml/strings_ops.yaml") +# The yaml file which include the python api info for ops +set(python_api_info_yaml_path + "${PADDLE_SOURCE_DIR}/paddle/phi/ops/yaml/python_api_info.yaml") message("Final State Eager CodeGen") add_custom_target( @@ -87,6 +90,7 @@ add_custom_target( "${PYTHON_EXECUTABLE}" "${PADDLE_SOURCE_DIR}/paddle/fluid/eager/auto_code_generator/generator/python_c_gen.py" "--api_yaml_path=${api_yaml_path},${fwd_api_yaml_path},${backward_yaml_path}" + "--python_api_info_yaml_path=${python_api_info_yaml_path}" "--source_path=${tmp_python_c_source_path}" "--header_path=${tmp_python_c_header_path}" COMMAND ${CMAKE_COMMAND} -E copy_if_different ${tmp_python_c_source_path} @@ -109,6 +113,7 @@ add_custom_target( "${PYTHON_EXECUTABLE}" "${PADDLE_SOURCE_DIR}/paddle/fluid/eager/auto_code_generator/generator/monkey_patch_gen.py" "--api_yaml_path=${ops_yaml_path}" + "--python_api_info_yaml_path=${python_api_info_yaml_path}" "--output_path=${tmp_monkey_patch_tensor_methods_path}" COMMAND ${CMAKE_COMMAND} -E copy_if_different diff --git a/paddle/fluid/eager/auto_code_generator/generator/codegen_utils.py b/paddle/fluid/eager/auto_code_generator/generator/codegen_utils.py index f2f8d8473da448..94114e804f2595 100644 --- a/paddle/fluid/eager/auto_code_generator/generator/codegen_utils.py +++ b/paddle/fluid/eager/auto_code_generator/generator/codegen_utils.py @@ -381,6 +381,32 @@ def ParseYamlReturns(string): return returns_list +def ParsePythonAPIInfoFromYAML(path) -> dict: + """ + Parse Python API information from a YAML file. + + Args: + path (str): The path to the YAML file. + + Returns: + dict: A dictionary containing Python API information, where the keys are operation names and the values are related api information. + + Raises: + RuntimeError: This exception is raised if an error occurs while parsing the YAML file. + """ + res_dict = {} + with open(path, "r", encoding="utf-8") as f: + try: + data = yaml.safe_load(f) + except yaml.YAMLError as e: + raise RuntimeError(f"read_python_api_info load error: {e}") + # Trans list to dict, the key is op in yaml item + for item in data: + if "op" in item.keys(): + res_dict.update({item["op"]: item}) + return res_dict + + def ParseYamlForwardFromBackward(string): # Example: matmul (const Tensor& x, const Tensor& y, bool transpose_x, bool transpose_y) -> Tensor(out) diff --git a/paddle/fluid/eager/auto_code_generator/generator/monkey_patch_gen.py b/paddle/fluid/eager/auto_code_generator/generator/monkey_patch_gen.py index ab2b7c6eed768c..1e42a97e5fad70 100644 --- a/paddle/fluid/eager/auto_code_generator/generator/monkey_patch_gen.py +++ b/paddle/fluid/eager/auto_code_generator/generator/monkey_patch_gen.py @@ -17,6 +17,7 @@ from codegen_utils import ( FunctionGeneratorBase, GeneratorBase, + ParsePythonAPIInfoFromYAML, ) IMPORT_TEMPLATE = """ @@ -29,12 +30,25 @@ def {func_name}(): """ -NAME_METHOD_MAPPING_TEMPLATE = """ ('{api_name}',_{api_name})""" +NAME_METHOD_MAPPING_TEMPLATE = """ ('{op_name}',_{op_name})""" METHODS_MAP_TEMPLATE = """ methods_map = [ {} ] + +""" +FUNCTIONS_MAP_TEMPLATE = """ +funcs_map = [ +{} +] + +""" +NN_FUNCTIONS_MAP_TEMPLATE = """ +nn_funcs_map = [ +{} +] + """ METHOD_TEMPLATE = """ @@ -42,13 +56,31 @@ def _{name}(*args, **kwargs): return _C_ops.{name}(*args, **kwargs) """ SET_METHOD_TEMPLATE = """ - # set methods for Tensor in dygraph + # set methods for paddle.Tensor in dygraph local_tensor = core.eager.Tensor for method_name, method in methods_map: setattr(local_tensor, method_name, method) + +""" +SET_FUNCTION_TEMPLATE = """ + # set functions for paddle + for method_name, method in funcs_map: setattr(paddle, method_name, method) """ +SET_NN_FUNCTION_TEMPLATE = """ + # set functions for paddle.nn.functional + for method_name, method in nn_funcs_map: + setattr(paddle.nn.functional, method_name, method) +""" +# The pair of name and func which should be added to paddle +paddle_func_map = [] +# The pair of name and func which should be added to paddle.Tensor +tensor_method_map = [] +# The pair of name and func which should be added to paddle.nn.functional +nn_func_map = [] +# The python api info which not in ops.yaml +python_api_info_from_yaml = {} class MethodGenerator(FunctionGeneratorBase): @@ -58,22 +90,40 @@ def __init__(self, forward_api_contents, namespace): # Generated Results self.Method_str = "" - def GenerateMethod(self, name): - self.Method_str = METHOD_TEMPLATE.format(name=name) - def run(self): # Initialized orig_forward_inputs_list, orig_forward_returns_list, orig_forward_attrs_list self.CollectOriginalForwardInfo() - if len(self.python_api_info) > 0: self.need_parse_python_api_args = True self.ParsePythonAPIInfo() - for name in self.python_api_names: - if "Tensor." in name: - api_name = name.split(".")[-1] - self.GenerateMethod(api_name) - self.api_name = api_name - break + self.Method_str = GenerateMethod(self.forward_api_name) + ClassifyAPIByPrefix(self.python_api_info, self.forward_api_name) + + +def ExtractPrefix(full_name): + res = "" + for m in full_name.split(".")[:-1]: + res += m + '.' + return res + + +def GenerateMethod(name): + return METHOD_TEMPLATE.format(name=name) + + +def ClassifyAPIByPrefix(python_api_info, op_name): + python_api_names = python_api_info["name"] + name_func_mapping = NAME_METHOD_MAPPING_TEMPLATE.format(op_name=op_name) + for name in python_api_names: + prefix = ExtractPrefix(name) + if prefix == "paddle.": + paddle_func_map.append(name_func_mapping) + elif prefix == "paddle.Tensor.": + tensor_method_map.append(name_func_mapping) + elif prefix == "paddle.nn.functional.": + nn_func_map.append(name_func_mapping) + else: + raise Exception("Unsupported Prefix " + prefix, "API : " + name) class MonkeyPatchTensorMethodsGenerator(GeneratorBase): @@ -92,23 +142,34 @@ def GenerateMonkeyPatchTensorMethods(self): forward_api_list = self.forward_api_list methods_map = [] # [("method_name",method),] + method_str = "" + # some python api info in ops.yaml for forward_api_content in forward_api_list: f_generator = MethodGenerator(forward_api_content, None) status = f_generator.run() - method_str = f_generator.Method_str - if method_str != "": - methods_map.append( - NAME_METHOD_MAPPING_TEMPLATE.format( - api_name=f_generator.api_name - ) - ) - self.MonkeyPatchTensorMethods_str += method_str - result = ',\n '.join(methods_map) + method_str += f_generator.Method_str + # some python api info not in ops.yaml but in python_api_info.yaml + for ops_name, python_api_info in python_api_info_from_yaml.items(): + method_str += GenerateMethod(ops_name) + ClassifyAPIByPrefix(python_api_info, ops_name) + + self.MonkeyPatchTensorMethods_str += method_str + result = ',\n '.join(tensor_method_map) self.MonkeyPatchTensorMethods_str += METHODS_MAP_TEMPLATE.format(result) + result = ',\n '.join(paddle_func_map) + self.MonkeyPatchTensorMethods_str += FUNCTIONS_MAP_TEMPLATE.format( + result + ) + result = ',\n '.join(nn_func_map) + self.MonkeyPatchTensorMethods_str += NN_FUNCTIONS_MAP_TEMPLATE.format( + result + ) self.MonkeyPatchTensorMethods_str += FUNCTION_NAME_TEMPLATE.format( func_name="monkey_patch_generated_methods_for_tensor" ) self.MonkeyPatchTensorMethods_str += SET_METHOD_TEMPLATE + self.MonkeyPatchTensorMethods_str += SET_FUNCTION_TEMPLATE + self.MonkeyPatchTensorMethods_str += SET_NN_FUNCTION_TEMPLATE def run(self): # Read Yaml file @@ -125,7 +186,7 @@ def ParseArguments(): ) parser.add_argument('--api_yaml_path', type=str) parser.add_argument('--output_path', type=str) - + parser.add_argument('--python_api_info_yaml_path', type=str) args = parser.parse_args() return args @@ -139,6 +200,12 @@ def GenerateMonkeyPathFile(filepath, python_c_str): args = ParseArguments() api_yaml_path = args.api_yaml_path output_path = args.output_path + python_api_info_yaml_path = args.python_api_info_yaml_path + + python_api_info_from_yaml = ParsePythonAPIInfoFromYAML( + python_api_info_yaml_path + ) + gen = MonkeyPatchTensorMethodsGenerator(api_yaml_path) gen.run() GenerateMonkeyPathFile(output_path, gen.MonkeyPatchTensorMethods_str) diff --git a/paddle/fluid/eager/auto_code_generator/generator/python_c_gen.py b/paddle/fluid/eager/auto_code_generator/generator/python_c_gen.py index 05e527e21be485..ffb2023b6bda64 100644 --- a/paddle/fluid/eager/auto_code_generator/generator/python_c_gen.py +++ b/paddle/fluid/eager/auto_code_generator/generator/python_c_gen.py @@ -21,6 +21,7 @@ GetForwardFunctionName, GetInplacedFunctionName, IsVectorTensorType, + ParsePythonAPIInfoFromYAML, ) args_default_mapping = { @@ -39,6 +40,8 @@ "multiply_grad", "pull_sparse_v2_grad", } +# The python api info which not in ops.yaml +python_api_info_from_yaml = {} def SkipAPIGeneration(forward_api_name): @@ -799,6 +802,16 @@ def pre_process_add_ampersand(s): # Generate Python-C Function Registration self.python_c_function_reg_str += python_c_inplace_func_reg_str + def InitAndParsePythonAPIInfo(self): + global python_api_info_from_yaml + if self.forward_api_name in python_api_info_from_yaml.keys(): + self.python_api_info = python_api_info_from_yaml[ + self.forward_api_name + ] + if len(self.python_api_info) > 0: + self.need_parse_python_api_args = True + self.ParsePythonAPIInfo() + def run(self, no_input_out_tensor=False): # Initialized is_forward_only self.CollectIsForwardOnly() @@ -811,11 +824,7 @@ def run(self, no_input_out_tensor=False): # Initialized orig_forward_inputs_list, orig_forward_returns_list, orig_forward_attrs_list self.CollectOriginalForwardInfo() - - if len(self.python_api_info) > 0: - self.need_parse_python_api_args = True - self.ParsePythonAPIInfo() - + self.InitAndParsePythonAPIInfo() if SkipAPIGeneration(self.forward_api_name): return False @@ -905,6 +914,7 @@ def ParseArguments(): description='Eager Code Generator Args Parser' ) parser.add_argument('--api_yaml_path', type=str) + parser.add_argument('--python_api_info_yaml_path', type=str) parser.add_argument('--source_path', type=str) parser.add_argument('--header_path', type=str) @@ -941,10 +951,14 @@ def GeneratePythonCFile(filepath, python_c_str): if __name__ == "__main__": args = ParseArguments() api_yaml_paths = args.api_yaml_path.split(",") - generated_python_c_functions = "" generated_python_c_registration = "" generated_python_c_functions_header = "" + python_api_info_yaml_path = args.python_api_info_yaml_path + + python_api_info_from_yaml = ParsePythonAPIInfoFromYAML( + python_api_info_yaml_path + ) for i in range(len(api_yaml_paths)): api_yaml_path = api_yaml_paths[i] @@ -970,7 +984,6 @@ def GeneratePythonCFile(filepath, python_c_str): python_c_str = GeneratePythonCWrappers( generated_python_c_functions, generated_python_c_registration ) - source_path = args.source_path header_path = args.header_path for path in [source_path, header_path]: diff --git a/paddle/fluid/pir/dialect/CMakeLists.txt b/paddle/fluid/pir/dialect/CMakeLists.txt index 31d7611f88c789..77d3bf03c9b767 100644 --- a/paddle/fluid/pir/dialect/CMakeLists.txt +++ b/paddle/fluid/pir/dialect/CMakeLists.txt @@ -193,6 +193,8 @@ set(python_c_source_file_tmp ${python_c_source_file}.tmp) set(trimmed_op_yaml_files ${op_fwd_yaml},${op_bwd_yaml},${fused_op_fwd_yaml},${fused_op_bwd_yaml},${pir_op_fwd_yaml},${pir_op_bwd_yaml},${pir_update_op_fwd_yaml},${pir_op_fwd_sparse_yaml},${pir_op_bfd_sparse_yaml} ) +set(python_api_info_yaml_path + "${PADDLE_SOURCE_DIR}/paddle/phi/ops/yaml/python_api_info.yaml") execute_process( COMMAND @@ -200,7 +202,8 @@ execute_process( ${trimmed_op_yaml_files} --op_compat_yaml_file ${op_compat_yaml_file} --namespaces "paddle,pybind" --python_c_def_h_file ${python_c_header_file_tmp} --python_c_def_cc_file - ${python_c_source_file_tmp}) + ${python_c_source_file_tmp} --python_api_info_yaml_path + ${python_api_info_yaml_path}) set(generated_files_python_c "${python_c_header_file}" "${python_c_source_file}") diff --git a/paddle/fluid/pir/dialect/op_generator/gen_utils.py b/paddle/fluid/pir/dialect/op_generator/gen_utils.py index 829d0835f675f0..0ce5c3d9705f93 100644 --- a/paddle/fluid/pir/dialect/op_generator/gen_utils.py +++ b/paddle/fluid/pir/dialect/op_generator/gen_utils.py @@ -11,6 +11,33 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import yaml + + +def ParsePythonAPIInfoFromYAML(path: str) -> dict: + """ + Parse Python API information from a YAML file. + + Args: + path (str): The path to the YAML file. + + Returns: + dict: A dictionary containing Python API information, where the keys are operation names and the values are related api information. + + Raises: + RuntimeError: This exception is raised if an error occurs while parsing the YAML file. + """ + res_dict = {} + with open(path, "r", encoding="utf-8") as f: + try: + data = yaml.safe_load(f) + except yaml.YAMLError as e: + raise RuntimeError(f"read_python_api_info load error: {e}") + # Trans list to dict, the key is op in yaml item + for item in data: + if "op" in item.keys(): + res_dict.update({item["op"]: item}) + return res_dict def to_pascal_case(s): diff --git a/paddle/fluid/pir/dialect/op_generator/python_c_gen.py b/paddle/fluid/pir/dialect/op_generator/python_c_gen.py index 7c60d327dc05f3..c869304d11d507 100644 --- a/paddle/fluid/pir/dialect/op_generator/python_c_gen.py +++ b/paddle/fluid/pir/dialect/op_generator/python_c_gen.py @@ -22,6 +22,7 @@ VECTOR_TYPE, CodeGen, ) +from gen_utils import ParsePythonAPIInfoFromYAML args_default_mapping = { "x": ["input"], @@ -29,6 +30,8 @@ "axis": ["dim"], "keepdims": ["keepdim"], } +# The python api info which not in ops.yaml +python_api_info_from_yaml = {} DISABLE_TIPS = ( "// This part of the function will be performed by a custom args mapper" ) @@ -698,6 +701,8 @@ def _gen_one_impl(self, op_info, op_name): self.need_parse_python_api_args = False self.use_custom_args_mapper = False + if op_name in python_api_info_from_yaml.keys(): + python_api_info = python_api_info_from_yaml[op_name] if python_api_info is not None: self.need_parse_python_api_args = True if "args_alias" in python_api_info.keys(): @@ -835,6 +840,7 @@ def ParseArguments(): ) parser.add_argument('--op_yaml_files', type=str) parser.add_argument('--op_compat_yaml_file', type=str) + parser.add_argument('--python_api_info_yaml_path', type=str) parser.add_argument('--namespaces', type=str) parser.add_argument('--python_c_def_h_file', type=str) parser.add_argument('--python_c_def_cc_file', type=str) @@ -845,6 +851,12 @@ def ParseArguments(): args = ParseArguments() op_yaml_files = args.op_yaml_files.split(",") op_compat_yaml_file = args.op_compat_yaml_file + + python_api_info_yaml_path = args.python_api_info_yaml_path + python_api_info_from_yaml = ParsePythonAPIInfoFromYAML( + python_api_info_yaml_path + ) + if args.namespaces is not None: namespaces = args.namespaces.split(",") python_c_def_h_file = args.python_c_def_h_file diff --git a/paddle/phi/ops/yaml/ops.yaml b/paddle/phi/ops/yaml/ops.yaml index e124d501d2a3b0..87f57dad551db2 100644 --- a/paddle/phi/ops/yaml/ops.yaml +++ b/paddle/phi/ops/yaml/ops.yaml @@ -243,10 +243,6 @@ - op : amax args : (Tensor x, int64_t[] axis={}, bool keepdim=false) - python_api : - name : [paddle.amax,paddle.Tensor.amax] - args_alias: - use_default_mapping : True output : Tensor(out) infer_meta : func : ReduceInferMeta @@ -257,10 +253,6 @@ - op : amin args : (Tensor x, int64_t[] axis={}, bool keepdim=false) - python_api : - name : [paddle.amin,paddle.Tensor.amin] - args_alias : - use_default_mapping : True output : Tensor(out) infer_meta : func : ReduceInferMeta diff --git a/paddle/phi/ops/yaml/python_api_info.yaml b/paddle/phi/ops/yaml/python_api_info.yaml new file mode 100644 index 00000000000000..740afa9ee689d0 --- /dev/null +++ b/paddle/phi/ops/yaml/python_api_info.yaml @@ -0,0 +1,9 @@ +- op : amin + name : [paddle.amin,paddle.Tensor.amin] + args_alias : + use_default_mapping : True + +- op : amax + name : [paddle.amax,paddle.Tensor.amax] + args_alias : + use_default_mapping : True