From 3c559f9d269f7818d00b6f0578c2b603afb72569 Mon Sep 17 00:00:00 2001 From: risemeup1 <62429225+risemeup1@users.noreply.github.com> Date: Sun, 19 Nov 2023 00:50:54 +0800 Subject: [PATCH 01/46] tesr (#59121) --- paddle/fluid/inference/lite/CMakeLists.txt | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/paddle/fluid/inference/lite/CMakeLists.txt b/paddle/fluid/inference/lite/CMakeLists.txt index 0167c8c3f656a3..a342b12f3456b8 100644 --- a/paddle/fluid/inference/lite/CMakeLists.txt +++ b/paddle/fluid/inference/lite/CMakeLists.txt @@ -14,17 +14,3 @@ cc_library( lite_tensor_utils SRCS tensor_utils.cc DEPS memcpy ${LITE_DEPS} framework_proto device_context ${XPU_DEPS}) -# TODO(shentanyue): fix ut later -# cc_test_old( -# test_lite_engine -# SRCS -# test_engine_lite.cc -# DEPS -# lite_engine -# protobuf -# framework_proto -# glog -# gtest -# analysis) -# cc_test_old(test_lite_tensor_utils SRCS test_tensor_utils.cc DEPS lite_engine -# lite_tensor_utils) From 175333ed99f9bf59258f8d5cc3f2b179cce1b4bc Mon Sep 17 00:00:00 2001 From: ShenLiang <1422485404@qq.com> Date: Sun, 19 Nov 2023 23:21:05 +0800 Subject: [PATCH 02/46] add gloabl shape in Dtensor (#59086) --- paddle/fluid/pybind/auto_parallel_py.cc | 2 -- paddle/fluid/pybind/eager_utils.cc | 1 - .../distributed/auto_parallel/dist_tensor.cc | 3 ++- .../distributed/auto_parallel/placement_types.h | 17 ++--------------- 4 files changed, 4 insertions(+), 19 deletions(-) diff --git a/paddle/fluid/pybind/auto_parallel_py.cc b/paddle/fluid/pybind/auto_parallel_py.cc index 3c06514a907da1..6401712457b348 100644 --- a/paddle/fluid/pybind/auto_parallel_py.cc +++ b/paddle/fluid/pybind/auto_parallel_py.cc @@ -90,7 +90,6 @@ using phi::distributed::auto_parallel::Machine; PyTypeObject *g_tensor_dist_attr_pytype = nullptr; PyTypeObject *g_dist_tensor_spec_pytype = nullptr; -PyTypeObject *g_placement_base_pytype = nullptr; PyTypeObject *g_process_mesh_pytype = nullptr; PyTypeObject *g_placement_shard_pytype = nullptr; PyTypeObject *g_placement_replicated_pytype = nullptr; @@ -413,7 +412,6 @@ void BindAutoParallel(py::module *m) { .def(py::self == py::self) .def(py::self != py::self); - g_placement_base_pytype = reinterpret_cast(Placement.ptr()); g_placement_shard_pytype = reinterpret_cast(Shard.ptr()); g_placement_replicated_pytype = reinterpret_cast(Replicate.ptr()); diff --git a/paddle/fluid/pybind/eager_utils.cc b/paddle/fluid/pybind/eager_utils.cc index 4aec990b566eb6..fc437327065644 100644 --- a/paddle/fluid/pybind/eager_utils.cc +++ b/paddle/fluid/pybind/eager_utils.cc @@ -66,7 +66,6 @@ extern PyTypeObject* g_framework_lodtensorarray_pytype; extern PyTypeObject* g_jit_function_pytype; extern PyTypeObject* g_tensor_dist_attr_pytype; extern PyTypeObject* g_process_mesh_pytype; -extern PyTypeObject* g_placement_base_pytype; extern PyTypeObject* g_placement_shard_pytype; extern PyTypeObject* g_placement_replicated_pytype; extern PyTypeObject* g_placement_partial_pytype; diff --git a/paddle/phi/core/distributed/auto_parallel/dist_tensor.cc b/paddle/phi/core/distributed/auto_parallel/dist_tensor.cc index 6c6ad522aeb409..0661ef17d2140c 100644 --- a/paddle/phi/core/distributed/auto_parallel/dist_tensor.cc +++ b/paddle/phi/core/distributed/auto_parallel/dist_tensor.cc @@ -70,7 +70,8 @@ DistTensor::DistTensor(const std::shared_ptr& global_value, DistTensor::DistTensor(const std::shared_ptr& global_value, const ProcessMesh& process_mesh, - const Placements& placements) { + const Placements& placements) + : dims_(global_value->dims()), value_(std::make_shared()) { dist_tensor_meta_ = DistTensorMeta( process_mesh, placements, diff --git a/paddle/phi/core/distributed/auto_parallel/placement_types.h b/paddle/phi/core/distributed/auto_parallel/placement_types.h index 67a0a6c51e1722..08e128d9c6f379 100644 --- a/paddle/phi/core/distributed/auto_parallel/placement_types.h +++ b/paddle/phi/core/distributed/auto_parallel/placement_types.h @@ -196,23 +196,10 @@ class DistTensorMeta : public std::enable_shared_from_this { namespace std { template <> -struct hash { - std::size_t operator()(const phi::distributed::Shard& p) const { +struct hash { + std::size_t operator()(const phi::distributed::Placement& p) const { return p.hash(); } }; -template <> -struct hash { - std::size_t operator()(const phi::distributed::Replicate& p) const { - return p.hash(); - } -}; - -template <> -struct hash { - std::size_t operator()(const phi::distributed::Partial& p) const { - return p.hash(); - } -}; } // namespace std From edd7b06499510a215e09164bf11d293d4c7252bd Mon Sep 17 00:00:00 2001 From: wanghuancoder Date: Mon, 20 Nov 2023 09:30:38 +0800 Subject: [PATCH 03/46] convert vector of tentor support disttensor convert (#59108) --- paddle/fluid/pybind/eager_utils.cc | 304 +++++++++++++++++++++++++---- 1 file changed, 265 insertions(+), 39 deletions(-) diff --git a/paddle/fluid/pybind/eager_utils.cc b/paddle/fluid/pybind/eager_utils.cc index fc437327065644..9cb81e16cfba4a 100644 --- a/paddle/fluid/pybind/eager_utils.cc +++ b/paddle/fluid/pybind/eager_utils.cc @@ -324,13 +324,28 @@ std::shared_ptr CastPyArg2JitFunction(PyObject* obj, std::vector CastPyArg2VectorOfTensor(PyObject* obj, ssize_t arg_pos) { std::vector result; + const phi::distributed::ProcessMesh* local_mesh = nullptr; + int mesh_start_index = -1; if (PyList_Check(obj)) { Py_ssize_t len = PyList_Size(obj); PyObject* item = nullptr; for (Py_ssize_t i = 0; i < len; i++) { item = PyList_GetItem(obj, i); if (PyObject_TypeCheck(item, p_tensor_type)) { - result.emplace_back(reinterpret_cast(item)->tensor); + paddle::Tensor& tensor = reinterpret_cast(item)->tensor; + if (local_mesh) { + ConvertToDistTensor(&tensor, local_mesh); + } else { + if (tensor.defined() && tensor.is_dist_tensor()) { + local_mesh = + &(std::dynamic_pointer_cast( + tensor.impl()) + ->dist_attr() + .process_mesh()); + mesh_start_index = i; + } + } + result.emplace_back(tensor); } else if (item == Py_None) { // emplace empty Tensor for None result.emplace_back(); @@ -343,13 +358,34 @@ std::vector CastPyArg2VectorOfTensor(PyObject* obj, i)); } } + for (Py_ssize_t i = 0; i < mesh_start_index; i++) { + item = PyList_GetItem(obj, i); + if (PyObject_TypeCheck(item, p_tensor_type)) { + paddle::Tensor& tensor = reinterpret_cast(item)->tensor; + ConvertToDistTensor(&tensor, local_mesh); + result[i] = tensor; + } + } } else if (PyTuple_Check(obj)) { Py_ssize_t len = PyTuple_Size(obj); PyObject* item = nullptr; for (Py_ssize_t i = 0; i < len; i++) { item = PyTuple_GetItem(obj, i); if (PyObject_TypeCheck(item, p_tensor_type)) { - result.emplace_back(reinterpret_cast(item)->tensor); + paddle::Tensor& tensor = reinterpret_cast(item)->tensor; + if (local_mesh) { + ConvertToDistTensor(&tensor, local_mesh); + } else { + if (tensor.defined() && tensor.is_dist_tensor()) { + local_mesh = + &(std::dynamic_pointer_cast( + tensor.impl()) + ->dist_attr() + .process_mesh()); + mesh_start_index = i; + } + } + result.emplace_back(tensor); } else if (item == Py_None) { // emplace empty Tensor for None result.emplace_back(); @@ -362,6 +398,14 @@ std::vector CastPyArg2VectorOfTensor(PyObject* obj, i)); } } + for (Py_ssize_t i = 0; i < mesh_start_index; i++) { + item = PyTuple_GetItem(obj, i); + if (PyObject_TypeCheck(item, p_tensor_type)) { + paddle::Tensor& tensor = reinterpret_cast(item)->tensor; + ConvertToDistTensor(&tensor, local_mesh); + result[i] = tensor; + } + } } else if (obj == Py_None) { return {}; } else if (PyObject_TypeCheck(obj, p_tensor_type)) { @@ -1310,6 +1354,8 @@ std::vector GetTensorListFromArgs( } std::vector result; + const phi::distributed::ProcessMesh* local_mesh = nullptr; + int mesh_start_index = -1; if (PyList_Check(list)) { Py_ssize_t len = PyList_Size(list); @@ -1323,13 +1369,27 @@ std::vector GetTensorListFromArgs( arg_idx)); } for (Py_ssize_t i = 0; i < len; i++) { - if (mesh) { - ConvertToDistTensor( - &(reinterpret_cast(PyList_GetItem(list, i))->tensor), - mesh); + paddle::Tensor& tensor = + reinterpret_cast(PyList_GetItem(list, i))->tensor; + if (local_mesh) { + ConvertToDistTensor(&tensor, local_mesh); + } else { + if (tensor.defined() && tensor.is_dist_tensor()) { + local_mesh = + &(std::dynamic_pointer_cast( + tensor.impl()) + ->dist_attr() + .process_mesh()); + mesh_start_index = i; + } } - result.emplace_back( - reinterpret_cast(PyList_GetItem(list, i))->tensor); + result.emplace_back(tensor); + } + for (Py_ssize_t i = 0; i < mesh_start_index; i++) { + paddle::Tensor& tensor = + reinterpret_cast(PyList_GetItem(list, i))->tensor; + ConvertToDistTensor(&tensor, local_mesh); + result[i] = tensor; } } else if (PyTuple_Check(list)) { Py_ssize_t len = PyTuple_Size(list); @@ -1343,14 +1403,27 @@ std::vector GetTensorListFromArgs( arg_idx)); } for (Py_ssize_t i = 0; i < len; i++) { - if (mesh) { - ConvertToDistTensor( - &(reinterpret_cast(PyTuple_GetItem(list, i)) - ->tensor), - mesh); + paddle::Tensor& tensor = + reinterpret_cast(PyTuple_GetItem(list, i))->tensor; + if (local_mesh) { + ConvertToDistTensor(&tensor, local_mesh); + } else { + if (tensor.defined() && tensor.is_dist_tensor()) { + local_mesh = + &(std::dynamic_pointer_cast( + tensor.impl()) + ->dist_attr() + .process_mesh()); + mesh_start_index = i; + } } - result.emplace_back( - reinterpret_cast(PyTuple_GetItem(list, i))->tensor); + result.emplace_back(tensor); + } + for (Py_ssize_t i = 0; i < mesh_start_index; i++) { + paddle::Tensor& tensor = + reinterpret_cast(PyTuple_GetItem(list, i))->tensor; + ConvertToDistTensor(&tensor, local_mesh); + result[i] = tensor; } } else if (list == Py_None) { return {}; @@ -1389,6 +1462,8 @@ paddle::optional> GetOptionalTensorListFromArgs( } std::vector result; + const phi::distributed::ProcessMesh* local_mesh = nullptr; + int mesh_start_index = -1; if (PyList_Check(list)) { Py_ssize_t len = PyList_Size(list); @@ -1402,13 +1477,27 @@ paddle::optional> GetOptionalTensorListFromArgs( arg_idx)); } for (Py_ssize_t i = 0; i < len; i++) { - if (mesh) { - ConvertToDistTensor( - &(reinterpret_cast(PyList_GetItem(list, i))->tensor), - mesh); + paddle::Tensor& tensor = + reinterpret_cast(PyList_GetItem(list, i))->tensor; + if (local_mesh) { + ConvertToDistTensor(&tensor, local_mesh); + } else { + if (tensor.defined() && tensor.is_dist_tensor()) { + local_mesh = + &(std::dynamic_pointer_cast( + tensor.impl()) + ->dist_attr() + .process_mesh()); + mesh_start_index = i; + } } - result.emplace_back( - reinterpret_cast(PyList_GetItem(list, i))->tensor); + result.emplace_back(tensor); + } + for (Py_ssize_t i = 0; i < mesh_start_index; i++) { + paddle::Tensor& tensor = + reinterpret_cast(PyList_GetItem(list, i))->tensor; + ConvertToDistTensor(&tensor, local_mesh); + result[i] = tensor; } } else if (PyTuple_Check(list)) { Py_ssize_t len = PyTuple_Size(list); @@ -1422,14 +1511,27 @@ paddle::optional> GetOptionalTensorListFromArgs( arg_idx)); } for (Py_ssize_t i = 0; i < len; i++) { - if (mesh) { - ConvertToDistTensor( - &(reinterpret_cast(PyTuple_GetItem(list, i)) - ->tensor), - mesh); + paddle::Tensor& tensor = + reinterpret_cast(PyTuple_GetItem(list, i))->tensor; + if (local_mesh) { + ConvertToDistTensor(&tensor, local_mesh); + } else { + if (tensor.defined() && tensor.is_dist_tensor()) { + local_mesh = + &(std::dynamic_pointer_cast( + tensor.impl()) + ->dist_attr() + .process_mesh()); + mesh_start_index = i; + } } - result.emplace_back( - reinterpret_cast(PyTuple_GetItem(list, i))->tensor); + result.emplace_back(tensor); + } + for (Py_ssize_t i = 0; i < mesh_start_index; i++) { + paddle::Tensor& tensor = + reinterpret_cast(PyTuple_GetItem(list, i))->tensor; + ConvertToDistTensor(&tensor, local_mesh); + result[i] = tensor; } } else { PADDLE_THROW(platform::errors::InvalidArgument( @@ -1500,6 +1602,8 @@ std::vector GetTensorPtrListFromArgs( } std::vector result; + const phi::distributed::ProcessMesh* local_mesh = nullptr; + int mesh_start_index = -1; if (PyList_Check(list)) { Py_ssize_t len = PyList_Size(list); @@ -1512,8 +1616,27 @@ std::vector GetTensorPtrListFromArgs( arg_idx)); } for (Py_ssize_t i = 0; i < len; i++) { - result.emplace_back( - &(reinterpret_cast(PyList_GetItem(list, i))->tensor)); + paddle::Tensor* tensor = + &(reinterpret_cast(PyList_GetItem(list, i))->tensor); + if (local_mesh) { + ConvertToDistTensor(tensor, local_mesh); + } else { + if (tensor->defined() && tensor->is_dist_tensor()) { + local_mesh = + &(std::dynamic_pointer_cast( + tensor->impl()) + ->dist_attr() + .process_mesh()); + mesh_start_index = i; + } + } + result.emplace_back(tensor); + } + for (Py_ssize_t i = 0; i < mesh_start_index; i++) { + paddle::Tensor* tensor = + &(reinterpret_cast(PyList_GetItem(list, i))->tensor); + ConvertToDistTensor(tensor, local_mesh); + result[i] = tensor; } } else if (PyTuple_Check(list)) { Py_ssize_t len = PyTuple_Size(list); @@ -1526,8 +1649,27 @@ std::vector GetTensorPtrListFromArgs( arg_idx)); } for (Py_ssize_t i = 0; i < len; i++) { - result.emplace_back( - &(reinterpret_cast(PyTuple_GetItem(list, i))->tensor)); + paddle::Tensor* tensor = + &(reinterpret_cast(PyTuple_GetItem(list, i))->tensor); + if (local_mesh) { + ConvertToDistTensor(tensor, local_mesh); + } else { + if (tensor->defined() && tensor->is_dist_tensor()) { + local_mesh = + &(std::dynamic_pointer_cast( + tensor->impl()) + ->dist_attr() + .process_mesh()); + mesh_start_index = i; + } + } + result.emplace_back(tensor); + } + for (Py_ssize_t i = 0; i < mesh_start_index; i++) { + paddle::Tensor* tensor = + &(reinterpret_cast(PyTuple_GetItem(list, i))->tensor); + ConvertToDistTensor(tensor, local_mesh); + result[i] = tensor; } } else if (list == Py_None) { return {}; @@ -1546,7 +1688,8 @@ std::vector GetTensorPtrListFromArgs( std::vector GetTensorPtrListFromPyObject(PyObject* obj) { std::vector result; - + const phi::distributed::ProcessMesh* local_mesh = nullptr; + int mesh_start_index = -1; if (PyList_Check(obj)) { Py_ssize_t len = PyList_Size(obj); if (len == 0) { @@ -1554,8 +1697,27 @@ std::vector GetTensorPtrListFromPyObject(PyObject* obj) { platform::errors::InvalidArgument("The list of Tensor is empty.")); } for (Py_ssize_t i = 0; i < len; i++) { - result.emplace_back( - &(reinterpret_cast(PyList_GetItem(obj, i))->tensor)); + paddle::Tensor* tensor = + &(reinterpret_cast(PyList_GetItem(obj, i))->tensor); + if (local_mesh) { + ConvertToDistTensor(tensor, local_mesh); + } else { + if (tensor->defined() && tensor->is_dist_tensor()) { + local_mesh = + &(std::dynamic_pointer_cast( + tensor->impl()) + ->dist_attr() + .process_mesh()); + mesh_start_index = i; + } + } + result.emplace_back(tensor); + } + for (Py_ssize_t i = 0; i < mesh_start_index; i++) { + paddle::Tensor* tensor = + &(reinterpret_cast(PyList_GetItem(obj, i))->tensor); + ConvertToDistTensor(tensor, local_mesh); + result[i] = tensor; } } else if (PyTuple_Check(obj)) { Py_ssize_t len = PyTuple_Size(obj); @@ -1564,8 +1726,27 @@ std::vector GetTensorPtrListFromPyObject(PyObject* obj) { platform::errors::InvalidArgument("The tuple of Tensor is empty.")); } for (Py_ssize_t i = 0; i < len; i++) { - result.emplace_back( - &(reinterpret_cast(PyTuple_GetItem(obj, i))->tensor)); + paddle::Tensor* tensor = + &(reinterpret_cast(PyTuple_GetItem(obj, i))->tensor); + if (local_mesh) { + ConvertToDistTensor(tensor, local_mesh); + } else { + if (tensor->defined() && tensor->is_dist_tensor()) { + local_mesh = + &(std::dynamic_pointer_cast( + tensor->impl()) + ->dist_attr() + .process_mesh()); + mesh_start_index = i; + } + } + result.emplace_back(tensor); + } + for (Py_ssize_t i = 0; i < mesh_start_index; i++) { + paddle::Tensor* tensor = + &(reinterpret_cast(PyTuple_GetItem(obj, i))->tensor); + ConvertToDistTensor(tensor, local_mesh); + result[i] = tensor; } } else { PADDLE_THROW(platform::errors::InvalidArgument( @@ -1580,13 +1761,29 @@ std::vector GetTensorPtrListFromPyObject(PyObject* obj) { std::vector GetTensorListFromPyObject(PyObject* obj, bool allow_none) { std::vector result; + const phi::distributed::ProcessMesh* local_mesh = nullptr; + int mesh_start_index = -1; + if (PyList_Check(obj)) { Py_ssize_t len = PyList_Size(obj); PyObject* item = nullptr; for (Py_ssize_t i = 0; i < len; i++) { item = PyList_GetItem(obj, i); if (PyObject_TypeCheck(item, p_tensor_type)) { - result.emplace_back(reinterpret_cast(item)->tensor); + paddle::Tensor& tensor = reinterpret_cast(item)->tensor; + if (local_mesh) { + ConvertToDistTensor(&tensor, local_mesh); + } else { + if (tensor.defined() && tensor.is_dist_tensor()) { + local_mesh = + &(std::dynamic_pointer_cast( + tensor.impl()) + ->dist_attr() + .process_mesh()); + mesh_start_index = i; + } + } + result.emplace_back(tensor); } else if (allow_none && (item == Py_None)) { VLOG(4) << "Got None in Tensor list: " << i; result.emplace_back(); @@ -1598,13 +1795,34 @@ std::vector GetTensorListFromPyObject(PyObject* obj, i)); } } + for (Py_ssize_t i = 0; i < mesh_start_index; i++) { + item = PyList_GetItem(obj, i); + if (PyObject_TypeCheck(item, p_tensor_type)) { + paddle::Tensor& tensor = reinterpret_cast(item)->tensor; + ConvertToDistTensor(&tensor, local_mesh); + result.emplace_back(tensor); + } + } } else if (PyTuple_Check(obj)) { Py_ssize_t len = PyTuple_Size(obj); PyObject* item = nullptr; for (Py_ssize_t i = 0; i < len; i++) { item = PyTuple_GetItem(obj, i); if (PyObject_TypeCheck(item, p_tensor_type)) { - result.emplace_back(reinterpret_cast(item)->tensor); + paddle::Tensor& tensor = reinterpret_cast(item)->tensor; + if (local_mesh) { + ConvertToDistTensor(&tensor, local_mesh); + } else { + if (tensor.defined() && tensor.is_dist_tensor()) { + local_mesh = + &(std::dynamic_pointer_cast( + tensor.impl()) + ->dist_attr() + .process_mesh()); + mesh_start_index = i; + } + } + result.emplace_back(tensor); } else if (allow_none && (item == Py_None)) { VLOG(4) << "Got None in Tensor list: " << i; result.emplace_back(); @@ -1616,6 +1834,14 @@ std::vector GetTensorListFromPyObject(PyObject* obj, i)); } } + for (Py_ssize_t i = 0; i < mesh_start_index; i++) { + item = PyTuple_GetItem(obj, i); + if (PyObject_TypeCheck(item, p_tensor_type)) { + paddle::Tensor& tensor = reinterpret_cast(item)->tensor; + ConvertToDistTensor(&tensor, local_mesh); + result.emplace_back(tensor); + } + } } else { PADDLE_THROW(platform::errors::InvalidArgument( "argument must be " From d2ebb839cd4709dba4d9cc607bf3f570f4eef2e7 Mon Sep 17 00:00:00 2001 From: Chen Zhiyang <1792266893@qq.com> Date: Mon, 20 Nov 2023 10:09:19 +0800 Subject: [PATCH 04/46] refine vjp gen to remove manual vjp for activation ops (#59104) --- .../fluid/pir/dialect/op_generator/op_gen.py | 23 +- .../dialect/op_generator/op_interface_gen.py | 16 +- .../pir/dialect/operator/ir/manual_op_vjp.cc | 220 ------------------ python/paddle/nn/functional/activation.py | 2 +- test/legacy_test/test_activation_nn_grad.py | 17 ++ test/legacy_test/test_mul_nn_grad.py | 10 +- test/legacy_test/test_nn_matmul_v2_grad.py | 177 +++++--------- 7 files changed, 111 insertions(+), 354 deletions(-) diff --git a/paddle/fluid/pir/dialect/op_generator/op_gen.py b/paddle/fluid/pir/dialect/op_generator/op_gen.py index 9e27095a73012a..abf2b5cd6cd1c0 100644 --- a/paddle/fluid/pir/dialect/op_generator/op_gen.py +++ b/paddle/fluid/pir/dialect/op_generator/op_gen.py @@ -218,11 +218,6 @@ class {op_name} : public pir::Op<{op_name}{interfaces}{traits}> {{ 'expand', } -vjp_manual_list = { - 'sigmoid_grad', - 'sigmoid_double_grad', -} - attr_types_map = { 'IntArray': ['paddle::dialect::IntArrayAttribute', 'IntArray'], 'Scalar': ['paddle::dialect::ScalarAttribute', 'Scalar'], @@ -423,6 +418,9 @@ def __init__(self, op_yaml_item, op_compat_item): # parse forward input name list and attribute name list self.forward_input_name_list = self.parse_forward_input_name() + # parse forward output name list + self.forward_output_name_list = self.parse_forward_output_name() + # parse traits list self.traits_list = self.parse_op_traits() @@ -446,6 +444,20 @@ def parse_forward_input_name(self): else: return None + def parse_forward_output_name(self): + if 'forward' in self.op_yaml_item: + forward_output_name_list = [] + forward_map = self.op_yaml_item['forward'] + if forward_map is not None: + outputs = forward_map['outputs'] + for output in outputs: + forward_output_name_list.append(output['name']) + return forward_output_name_list + else: + return None + else: + return None + def cross_check(self, name_list, type_list, optional_list=None): assert len(name_list) == len( type_list @@ -1595,7 +1607,6 @@ def OpGenerator( op_info.backward_name and op_info.op_phi_name[0] not in vjp_interface_black_list - and op_info.op_phi_name[0] not in vjp_manual_list ): op_vjp_str = gen_op_vjp_str( op_class_name, diff --git a/paddle/fluid/pir/dialect/op_generator/op_interface_gen.py b/paddle/fluid/pir/dialect/op_generator/op_interface_gen.py index 6d7c5224e3803e..92881b5d48523d 100644 --- a/paddle/fluid/pir/dialect/op_generator/op_interface_gen.py +++ b/paddle/fluid/pir/dialect/op_generator/op_interface_gen.py @@ -122,6 +122,14 @@ def gen_op_vjp_str( fwd_input_and_mutable_attr_name_list = ( op_info.input_name_list + op_info.mutable_attribute_name_list ) + if op_grad_info.forward_input_name_list: + fwd_inputs_list = op_grad_info.forward_input_name_list + else: + fwd_inputs_list = fwd_input_and_mutable_attr_name_list + if op_grad_info.forward_output_name_list: + fwd_outputs_list = op_grad_info.forward_output_name_list + else: + fwd_outputs_list = op_info.output_name_list backward_input_code = '' build_args_str = '' @@ -133,12 +141,12 @@ def gen_op_vjp_str( vjp_param_name = '' index_0 = -1 - if bw_input_name in fwd_input_and_mutable_attr_name_list: + if bw_input_name in fwd_inputs_list: vjp_param_name = 'inputs_' - index_0 = fwd_input_and_mutable_attr_name_list.index(bw_input_name) - elif bw_input_name in op_info.output_name_list: + index_0 = fwd_inputs_list.index(bw_input_name) + elif bw_input_name in fwd_outputs_list: vjp_param_name = 'outputs' - index_0 = op_info.output_name_list.index(bw_input_name) + index_0 = fwd_outputs_list.index(bw_input_name) else: vjp_param_name = 'out_grads' grad_idx += 1 diff --git a/paddle/fluid/pir/dialect/operator/ir/manual_op_vjp.cc b/paddle/fluid/pir/dialect/operator/ir/manual_op_vjp.cc index 6cf4c211348a57..a37cbd681d185d 100644 --- a/paddle/fluid/pir/dialect/operator/ir/manual_op_vjp.cc +++ b/paddle/fluid/pir/dialect/operator/ir/manual_op_vjp.cc @@ -127,225 +127,5 @@ std::vector> ExpandOp::Vjp( return res; } -std::vector> SigmoidDoubleGradOp::Vjp( - pir::Operation* op, - const std::vector>& inputs_, - const std::vector>& outputs, - const std::vector>& out_grads, - const std::vector>& stop_gradients) { - PADDLE_ENFORCE_EQ( - inputs_.size(), - 3, - platform::errors::InvalidArgument( - "sigmoid_double_grad op's inputs size should be 3, but now is %d.", - inputs_.size())); - PADDLE_ENFORCE_EQ( - outputs.size(), - 2, - platform::errors::InvalidArgument( - "sigmoid_double_grad op's outputs size should be 2, but now is %d.", - outputs.size())); - - VLOG(6) << "Prepare inputs of sigmoid_triple_grad"; - - Tensor out(std::make_shared(inputs_[0][0])); - Tensor out_grad(std::make_shared(inputs_[1][0])); - Tensor grad_grad_x(std::make_shared(inputs_[2][0])); - Tensor grad_out_grad( - std::make_shared(out_grads[0][0])); - paddle::optional grad_grad_out_grad; - if (!IsEmptyValue(out_grads[1][0])) { - grad_grad_out_grad = paddle::make_optional( - Tensor(std::make_shared(out_grads[1][0]))); - } - - VLOG(6) << "Vjp prepare Prepare attributes of sigmoid_triple_grad"; - - VLOG(6) << "Vjp prepare call sigmoid_double_grad's vjp inteface"; - - std::vector> tensor_res = - primitive::sigmoid_double_grad_vjp(out, - out_grad, - grad_grad_x, - grad_out_grad, - grad_grad_out_grad, - stop_gradients); - - VLOG(6) << "Vjp prepare stop gradient of sigmoid_triple_grad"; - - std::vector> res(tensor_res.size()); - for (size_t i = 0; i < tensor_res.size(); ++i) { - res[i].resize(tensor_res[i].size()); - for (size_t j = 0; j < tensor_res[i].size(); ++j) { - if (tensor_res[i][j].defined()) { - res[i][j] = std::static_pointer_cast( - tensor_res[i][j].impl()) - ->value() - .dyn_cast(); - } - } - } - return res; -} - -std::vector> SigmoidDoubleGrad_Op::Vjp( - pir::Operation* op, - const std::vector>& inputs_, - const std::vector>& outputs, - const std::vector>& out_grads, - const std::vector>& stop_gradients) { - PADDLE_ENFORCE_EQ( - inputs_.size(), - 3, - platform::errors::InvalidArgument( - "sigmoid_double_grad op's inputs size should be 3, but now is %d.", - inputs_.size())); - PADDLE_ENFORCE_EQ( - outputs.size(), - 2, - platform::errors::InvalidArgument( - "sigmoid_double_grad op's outputs size should be 2, but now is %d.", - outputs.size())); - - VLOG(6) << "Prepare inputs of sigmoid_triple_grad"; - - Tensor out(std::make_shared(inputs_[0][0])); - Tensor out_grad(std::make_shared(inputs_[1][0])); - Tensor grad_grad_x(std::make_shared(inputs_[2][0])); - Tensor grad_out_grad( - std::make_shared(out_grads[0][0])); - paddle::optional grad_grad_out_grad; - if (!IsEmptyValue(out_grads[1][0])) { - grad_grad_out_grad = paddle::make_optional( - Tensor(std::make_shared(out_grads[1][0]))); - } - - VLOG(6) << "Vjp prepare Prepare attributes of sigmoid_triple_grad"; - - VLOG(6) << "Vjp prepare call sigmoid_double_grad's vjp inteface"; - - std::vector> tensor_res = - primitive::sigmoid_double_grad_vjp(out, - out_grad, - grad_grad_x, - grad_out_grad, - grad_grad_out_grad, - stop_gradients); - - VLOG(6) << "Vjp prepare stop gradient of sigmoid_triple_grad"; - - std::vector> res(tensor_res.size()); - for (size_t i = 0; i < tensor_res.size(); ++i) { - res[i].resize(tensor_res[i].size()); - for (size_t j = 0; j < tensor_res[i].size(); ++j) { - if (tensor_res[i][j].defined()) { - res[i][j] = std::static_pointer_cast( - tensor_res[i][j].impl()) - ->value() - .dyn_cast(); - } - } - } - return res; -} - -std::vector> SigmoidGradOp::Vjp( - pir::Operation* op, - const std::vector>& inputs_, - const std::vector>& outputs, - const std::vector>& out_grads, - const std::vector>& stop_gradients) { - PADDLE_ENFORCE_EQ( - inputs_.size(), - 2, - platform::errors::InvalidArgument( - "sigmoid_grad op's inputs size should be 2, but now is %d.", - inputs_.size())); - PADDLE_ENFORCE_EQ( - outputs.size(), - 1, - platform::errors::InvalidArgument( - "sigmoid_grad op's outputs size should be 1, but now is %d.", - outputs.size())); - - VLOG(6) << "Prepare inputs of sigmoid_double_grad"; - - Tensor out(std::make_shared(inputs_[0][0])); - Tensor fwd_grad_out(std::make_shared(inputs_[1][0])); - Tensor grad_x_grad(std::make_shared(out_grads[0][0])); - - VLOG(6) << "Vjp prepare Prepare attributes of sigmoid_double_grad"; - - VLOG(6) << "Vjp prepare call sigmoid_grad's vjp inteface"; - - std::vector> tensor_res = primitive::sigmoid_grad_vjp( - out, fwd_grad_out, grad_x_grad, stop_gradients); - - VLOG(6) << "Vjp prepare stop gradient of sigmoid_double_grad"; - - std::vector> res(tensor_res.size()); - for (size_t i = 0; i < tensor_res.size(); ++i) { - res[i].resize(tensor_res[i].size()); - for (size_t j = 0; j < tensor_res[i].size(); ++j) { - if (tensor_res[i][j].defined()) { - res[i][j] = std::static_pointer_cast( - tensor_res[i][j].impl()) - ->value() - .dyn_cast(); - } - } - } - return res; -} - -std::vector> SigmoidGrad_Op::Vjp( - pir::Operation* op, - const std::vector>& inputs_, - const std::vector>& outputs, - const std::vector>& out_grads, - const std::vector>& stop_gradients) { - PADDLE_ENFORCE_EQ( - inputs_.size(), - 2, - platform::errors::InvalidArgument( - "sigmoid_grad op's inputs size should be 2, but now is %d.", - inputs_.size())); - PADDLE_ENFORCE_EQ( - outputs.size(), - 1, - platform::errors::InvalidArgument( - "sigmoid_grad op's outputs size should be 1, but now is %d.", - outputs.size())); - - VLOG(6) << "Prepare inputs of sigmoid_double_grad"; - - Tensor out(std::make_shared(inputs_[0][0])); - Tensor fwd_grad_out(std::make_shared(inputs_[1][0])); - Tensor grad_x_grad(std::make_shared(out_grads[0][0])); - - VLOG(6) << "Vjp prepare Prepare attributes of sigmoid_double_grad"; - - VLOG(6) << "Vjp prepare call sigmoid_grad_'s vjp inteface"; - - std::vector> tensor_res = primitive::sigmoid_grad_vjp( - out, fwd_grad_out, grad_x_grad, stop_gradients); - - VLOG(6) << "Vjp prepare stop gradient of sigmoid_double_grad"; - - std::vector> res(tensor_res.size()); - for (size_t i = 0; i < tensor_res.size(); ++i) { - res[i].resize(tensor_res[i].size()); - for (size_t j = 0; j < tensor_res[i].size(); ++j) { - if (tensor_res[i][j].defined()) { - res[i][j] = std::static_pointer_cast( - tensor_res[i][j].impl()) - ->value() - .dyn_cast(); - } - } - } - return res; -} - } // namespace dialect } // namespace paddle diff --git a/python/paddle/nn/functional/activation.py b/python/paddle/nn/functional/activation.py index a4c80da11957b7..0cda9a1e7480c8 100644 --- a/python/paddle/nn/functional/activation.py +++ b/python/paddle/nn/functional/activation.py @@ -113,7 +113,7 @@ def elu(x, alpha=1.0, name=None): [ 1. , 15.60000038]]) """ - if in_dynamic_mode(): + if in_dynamic_or_pir_mode(): return _C_ops.elu(x, alpha) else: diff --git a/test/legacy_test/test_activation_nn_grad.py b/test/legacy_test/test_activation_nn_grad.py index 0c0c76e24b87ff..8203206d1c77c5 100644 --- a/test/legacy_test/test_activation_nn_grad.py +++ b/test/legacy_test/test_activation_nn_grad.py @@ -155,6 +155,7 @@ class TestAbsDoubleGradCheck(unittest.TestCase): def abs_wrapper(self, x): return paddle.abs(x[0]) + @test_with_pir_api @prog_scope() def func(self, place): shape = [2, 3, 7, 9] @@ -182,6 +183,7 @@ def test_grad(self): class TestReluDoubleGradCheck(unittest.TestCase): + @test_with_pir_api @prog_scope() def func(self, place): shape = [2, 3, 7, 9] @@ -211,6 +213,7 @@ class TestLeakyReluDoubleGradCheck(unittest.TestCase): def leaky_relu_wrapper(self, x): return paddle.nn.functional.leaky_relu(x[0], negative_slope=0.2) + @test_with_pir_api @prog_scope() def func(self, place): shape = [2, 3, 7, 9] @@ -245,6 +248,7 @@ class TestELUDoubleGradCheck(unittest.TestCase): def elu_wrapper(self, x): return paddle.nn.functional.elu(x[0], alpha=0.2) + @test_with_pir_api @prog_scope() def func(self, place): shape = [2, 4, 4, 4] @@ -279,6 +283,7 @@ class TestCELUDoubleGradCheck(unittest.TestCase): def celu_wrapper(self, x): return paddle.nn.functional.celu(x[0], alpha=0.2) + @test_with_pir_api @prog_scope() def func(self, place): shape = [2, 4, 4, 4] @@ -313,6 +318,7 @@ class TestSoftplusDoubleGradCheck(unittest.TestCase): def softplus_wrapper(self, x): return F.softplus(x[0], beta=1, threshold=20) + @test_with_pir_api @prog_scope() def func(self, place): shape = [2, 4, 4, 4] @@ -348,6 +354,7 @@ class TestSqrtDoubleGradCheck(unittest.TestCase): def sqrt_wrapper(self, x): return paddle.sqrt(x[0]) + @test_with_pir_api @prog_scope() def func(self, place): shape = [2, 3, 7, 9] @@ -380,6 +387,7 @@ class TestRsqrtDoubleGradCheck(unittest.TestCase): def rsqrt_wrapper(self, x): return paddle.rsqrt(x[0]) + @test_with_pir_api @prog_scope() def func(self, place): shape = [2, 3, 7, 9] @@ -412,6 +420,7 @@ class TestSquareDoubleGradCheck(unittest.TestCase): def square_wrapper(self, x): return paddle.square(x[0]) + @test_with_pir_api @prog_scope() def func(self, place): # the shape of input variable should be clearly specified, not include -1. @@ -444,6 +453,7 @@ class TestLogDoubleGradCheck(unittest.TestCase): def log_wrapper(self, x): return paddle.log(x[0]) + @test_with_pir_api @prog_scope() def func(self, place): shape = [2, 3, 7, 9] @@ -476,6 +486,7 @@ class TestSinDoubleGradCheck(unittest.TestCase): def sin_wrapper(self, x): return paddle.sin(x[0]) + @test_with_pir_api @prog_scope() def func(self, place): shape = [2, 3, 7, 9] @@ -506,6 +517,7 @@ class TestCosDoubleGradCheck(unittest.TestCase): def cos_wrapper(self, x): return paddle.cos(x[0]) + @test_with_pir_api @prog_scope() def func(self, place): shape = [2, 3, 7, 9] @@ -594,6 +606,7 @@ class TestSinTripleGradCheck(unittest.TestCase): def sin_wrapper(self, x): return paddle.sin(x[0]) + @test_with_pir_api @prog_scope() def func(self, place): shape = [2, 3, 7, 9] @@ -624,6 +637,7 @@ class TestPowTripleGradCheck1(unittest.TestCase): def pow_wrapper(self, x): return paddle.pow(x[0], 1) + @test_with_pir_api @prog_scope() def func(self, place): shape = [2, 3, 7, 9] @@ -653,6 +667,7 @@ class TestPowTripleGradCheck2(unittest.TestCase): def pow_wrapper(self, x): return paddle.pow(x[0], 2) + @test_with_pir_api @prog_scope() def func(self, place): shape = [2, 3, 7, 9] @@ -682,6 +697,7 @@ class TestPowTripleGradCheck3(unittest.TestCase): def pow_wrapper(self, x): return paddle.pow(x[0], 4) + @test_with_pir_api @prog_scope() def func(self, place): shape = [2, 3, 7, 9] @@ -711,6 +727,7 @@ class TestCosTripleGradCheck(unittest.TestCase): def cos_wrapper(self, x): return paddle.cos(x[0]) + @test_with_pir_api @prog_scope() def func(self, place): shape = [2, 3, 7, 9] diff --git a/test/legacy_test/test_mul_nn_grad.py b/test/legacy_test/test_mul_nn_grad.py index eb30a0e1a50688..41a3ac2d9a03c7 100644 --- a/test/legacy_test/test_mul_nn_grad.py +++ b/test/legacy_test/test_mul_nn_grad.py @@ -21,6 +21,7 @@ import paddle from paddle import base from paddle.base import core +from paddle.pir_utils import test_with_pir_api paddle.enable_static() @@ -35,17 +36,14 @@ def init_test(self): self.transpose_x = False self.transpose_y = False + @test_with_pir_api @prog_scope() def func(self, place): eps = 0.005 dtype = np.float64 typename = "float64" - x = paddle.create_parameter( - dtype=typename, shape=self.x_shape, name='x' - ) - y = paddle.create_parameter( - dtype=typename, shape=self.y_shape, name='y' - ) + x = paddle.static.data(dtype=typename, shape=self.x_shape, name='x') + y = paddle.static.data(dtype=typename, shape=self.y_shape, name='y') out = paddle.matmul( x, y, self.transpose_x, self.transpose_y, name='out' ) diff --git a/test/legacy_test/test_nn_matmul_v2_grad.py b/test/legacy_test/test_nn_matmul_v2_grad.py index 62350f9532ea1d..338eb224166bfa 100644 --- a/test/legacy_test/test_nn_matmul_v2_grad.py +++ b/test/legacy_test/test_nn_matmul_v2_grad.py @@ -21,6 +21,7 @@ import paddle from paddle import base from paddle.base import core +from paddle.pir_utils import test_with_pir_api paddle.enable_static() @@ -35,17 +36,14 @@ def init_test(self): self.transpose_x = False self.transpose_y = False + @test_with_pir_api @prog_scope() def func(self, place): eps = 0.005 dtype = np.float64 typename = "float64" - x = paddle.static.create_parameter( - dtype=typename, shape=self.x_shape, name='x' - ) - y = paddle.static.create_parameter( - dtype=typename, shape=self.y_shape, name='y' - ) + x = paddle.static.data(dtype=typename, shape=self.x_shape, name='x') + y = paddle.static.data(dtype=typename, shape=self.y_shape, name='y') out = paddle.matmul( x, y, self.transpose_x, self.transpose_y, name='out' ) @@ -89,17 +87,14 @@ def init_test(self): self.transpose_x = True self.transpose_y = False + @test_with_pir_api @prog_scope() def func(self, place): eps = 0.005 dtype = np.float64 typename = "float64" - x = paddle.static.create_parameter( - dtype=typename, shape=self.x_shape, name='x' - ) - y = paddle.static.create_parameter( - dtype=typename, shape=self.y_shape, name='y' - ) + x = paddle.static.data(dtype=typename, shape=self.x_shape, name='x') + y = paddle.static.data(dtype=typename, shape=self.y_shape, name='y') out = paddle.matmul( x, y, self.transpose_x, self.transpose_y, name='out' ) @@ -128,17 +123,14 @@ def init_test(self): self.transpose_x = False self.transpose_y = False + @test_with_pir_api @prog_scope() def func(self, place): eps = 0.005 dtype = np.float64 typename = "float64" - x = paddle.static.create_parameter( - dtype=typename, shape=self.x_shape, name='x' - ) - y = paddle.static.create_parameter( - dtype=typename, shape=self.y_shape, name='y' - ) + x = paddle.static.data(dtype=typename, shape=self.x_shape, name='x') + y = paddle.static.data(dtype=typename, shape=self.y_shape, name='y') out = paddle.matmul( x, y, self.transpose_x, self.transpose_y, name='out' ) @@ -161,40 +153,35 @@ class TestMatmulTripleGradCheckDotCase(unittest.TestCase): def setUp(self): self.init_test() + def init_test(self): + self.x_shape = [2] + self.y_shape = [2] + self.transpose_x = False + self.transpose_y = False -def init_test(self): - self.x_shape = [2] - self.y_shape = [2] - self.transpose_x = False - self.transpose_y = False - - -@prog_scope() -def func(self, place): - eps = 0.005 - dtype = np.float64 - typename = "float64" - x = paddle.static.create_parameter( - dtype=typename, shape=self.x_shape, name='x' - ) - y = paddle.static.create_parameter( - dtype=typename, shape=self.y_shape, name='y' - ) - out = paddle.matmul(x, y, self.transpose_x, self.transpose_y, name='out') - np.random.seed(2021) - x_arr = np.random.uniform(-1, 1, self.x_shape).astype(dtype) - y_arr = np.random.uniform(-1, 1, self.y_shape).astype(dtype) - gradient_checker.triple_grad_check( - [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps - ) - - -def test_grad(self): - places = [base.CPUPlace()] - if core.is_compiled_with_cuda(): - places.append(base.CUDAPlace(0)) - for p in places: - self.func(p) + @prog_scope() + def func(self, place): + eps = 0.005 + dtype = np.float64 + typename = "float64" + x = paddle.static.data(dtype=typename, shape=self.x_shape, name='x') + y = paddle.static.data(dtype=typename, shape=self.y_shape, name='y') + out = paddle.matmul( + x, y, self.transpose_x, self.transpose_y, name='out' + ) + np.random.seed(2021) + x_arr = np.random.uniform(-1, 1, self.x_shape).astype(dtype) + y_arr = np.random.uniform(-1, 1, self.y_shape).astype(dtype) + gradient_checker.triple_grad_check( + [x, y], out, x_init=[x_arr, y_arr], place=place, eps=eps + ) + + def test_grad(self): + places = [base.CPUPlace()] + if core.is_compiled_with_cuda(): + places.append(base.CUDAPlace(0)) + for p in places: + self.func(p) class TestMatmulTripleGradCheckNormalCase1(unittest.TestCase): @@ -212,12 +199,8 @@ def func(self, place): eps = 0.005 dtype = np.float64 typename = "float64" - x = paddle.static.create_parameter( - dtype=typename, shape=self.x_shape, name='x' - ) - y = paddle.static.create_parameter( - dtype=typename, shape=self.y_shape, name='y' - ) + x = paddle.static.data(dtype=typename, shape=self.x_shape, name='x') + y = paddle.static.data(dtype=typename, shape=self.y_shape, name='y') out = paddle.matmul( x, y, self.transpose_x, self.transpose_y, name='out' ) @@ -251,12 +234,8 @@ def func(self, place): eps = 0.005 dtype = np.float64 typename = "float64" - x = paddle.static.create_parameter( - dtype=typename, shape=self.x_shape, name='x' - ) - y = paddle.static.create_parameter( - dtype=typename, shape=self.y_shape, name='y' - ) + x = paddle.static.data(dtype=typename, shape=self.x_shape, name='x') + y = paddle.static.data(dtype=typename, shape=self.y_shape, name='y') out = paddle.matmul( x, y, self.transpose_x, self.transpose_y, name='out' ) @@ -290,12 +269,8 @@ def func(self, place): eps = 0.005 dtype = np.float64 typename = "float64" - x = paddle.static.create_parameter( - dtype=typename, shape=self.x_shape, name='x' - ) - y = paddle.static.create_parameter( - dtype=typename, shape=self.y_shape, name='y' - ) + x = paddle.static.data(dtype=typename, shape=self.x_shape, name='x') + y = paddle.static.data(dtype=typename, shape=self.y_shape, name='y') out = paddle.matmul( x, y, self.transpose_x, self.transpose_y, name='out' ) @@ -329,12 +304,8 @@ def func(self, place): eps = 0.005 dtype = np.float64 typename = "float64" - x = paddle.static.create_parameter( - dtype=typename, shape=self.x_shape, name='x' - ) - y = paddle.static.create_parameter( - dtype=typename, shape=self.y_shape, name='y' - ) + x = paddle.static.data(dtype=typename, shape=self.x_shape, name='x') + y = paddle.static.data(dtype=typename, shape=self.y_shape, name='y') out = paddle.matmul( x, y, self.transpose_x, self.transpose_y, name='out' ) @@ -368,12 +339,8 @@ def func(self, place): eps = 0.005 dtype = np.float64 typename = "float64" - x = paddle.static.create_parameter( - dtype=typename, shape=self.x_shape, name='x' - ) - y = paddle.static.create_parameter( - dtype=typename, shape=self.y_shape, name='y' - ) + x = paddle.static.data(dtype=typename, shape=self.x_shape, name='x') + y = paddle.static.data(dtype=typename, shape=self.y_shape, name='y') out = paddle.matmul( x, y, self.transpose_x, self.transpose_y, name='out' ) @@ -407,12 +374,8 @@ def func(self, place): eps = 0.005 dtype = np.float64 typename = "float64" - x = paddle.static.create_parameter( - dtype=typename, shape=self.x_shape, name='x' - ) - y = paddle.static.create_parameter( - dtype=typename, shape=self.y_shape, name='y' - ) + x = paddle.static.data(dtype=typename, shape=self.x_shape, name='x') + y = paddle.static.data(dtype=typename, shape=self.y_shape, name='y') out = paddle.matmul( x, y, self.transpose_x, self.transpose_y, name='out' ) @@ -446,12 +409,8 @@ def func(self, place): eps = 0.005 dtype = np.float64 typename = "float64" - x = paddle.static.create_parameter( - dtype=typename, shape=self.x_shape, name='x' - ) - y = paddle.static.create_parameter( - dtype=typename, shape=self.y_shape, name='y' - ) + x = paddle.static.data(dtype=typename, shape=self.x_shape, name='x') + y = paddle.static.data(dtype=typename, shape=self.y_shape, name='y') out = paddle.matmul( x, y, self.transpose_x, self.transpose_y, name='out' ) @@ -485,12 +444,8 @@ def func(self, place): eps = 0.005 dtype = np.float64 typename = "float64" - x = paddle.static.create_parameter( - dtype=typename, shape=self.x_shape, name='x' - ) - y = paddle.static.create_parameter( - dtype=typename, shape=self.y_shape, name='y' - ) + x = paddle.static.data(dtype=typename, shape=self.x_shape, name='x') + y = paddle.static.data(dtype=typename, shape=self.y_shape, name='y') out = paddle.matmul( x, y, self.transpose_x, self.transpose_y, name='out' ) @@ -524,12 +479,8 @@ def func(self, place): eps = 0.005 dtype = np.float64 typename = "float64" - x = paddle.static.create_parameter( - dtype=typename, shape=self.x_shape, name='x' - ) - y = paddle.static.create_parameter( - dtype=typename, shape=self.y_shape, name='y' - ) + x = paddle.static.data(dtype=typename, shape=self.x_shape, name='x') + y = paddle.static.data(dtype=typename, shape=self.y_shape, name='y') out = paddle.matmul( x, y, self.transpose_x, self.transpose_y, name='out' ) @@ -563,12 +514,8 @@ def func(self, place): eps = 0.005 dtype = np.float64 typename = "float64" - x = paddle.static.create_parameter( - dtype=typename, shape=self.x_shape, name='x' - ) - y = paddle.static.create_parameter( - dtype=typename, shape=self.y_shape, name='y' - ) + x = paddle.static.data(dtype=typename, shape=self.x_shape, name='x') + y = paddle.static.data(dtype=typename, shape=self.y_shape, name='y') out = paddle.matmul( x, y, self.transpose_x, self.transpose_y, name='out' ) @@ -602,12 +549,8 @@ def func(self, place): eps = 0.005 dtype = np.float64 typename = "float64" - x = paddle.static.create_parameter( - dtype=typename, shape=self.x_shape, name='x' - ) - y = paddle.static.create_parameter( - dtype=typename, shape=self.y_shape, name='y' - ) + x = paddle.static.data(dtype=typename, shape=self.x_shape, name='x') + y = paddle.static.data(dtype=typename, shape=self.y_shape, name='y') out = paddle.matmul( x, y, self.transpose_x, self.transpose_y, name='out' ) From 483976620611c85a47d3ab781899d33d490cc3d9 Mon Sep 17 00:00:00 2001 From: Lu Qi <61354321+MarioLulab@users.noreply.github.com> Date: Mon, 20 Nov 2023 10:27:45 +0800 Subject: [PATCH 05/46] [pir] Add pybind property id of OpResult (#59064) * add OpResult pybind id * remove startswith 0x --- paddle/fluid/pybind/pir.cc | 14 ++++++++++++++ test/ir/pir/test_ir_pybind.py | 8 ++++++++ 2 files changed, 22 insertions(+) diff --git a/paddle/fluid/pybind/pir.cc b/paddle/fluid/pybind/pir.cc index e4f597cb6e980a..41de635ce9a551 100644 --- a/paddle/fluid/pybind/pir.cc +++ b/paddle/fluid/pybind/pir.cc @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -746,6 +747,19 @@ void BindOpResult(py::module *m) { "persistable")); } }) + .def_property_readonly( + "id", + [](OpResult &self) { + if (self.impl() == nullptr) { + PADDLE_THROW(phi::errors::InvalidArgument( + "Currently, we can only get id of OpResult whose impl " + "is not nullptr")); + } else { + std::stringstream ss; + ss << std::hex << self.impl(); + return ss.str(); + } + }) .def("initialized", [](OpResult &self) { if (self.impl() == nullptr || self.type().storage() == nullptr) { diff --git a/test/ir/pir/test_ir_pybind.py b/test/ir/pir/test_ir_pybind.py index 4c42c0f6f77ae8..b7a38ed8a359ce 100644 --- a/test/ir/pir/test_ir_pybind.py +++ b/test/ir/pir/test_ir_pybind.py @@ -205,6 +205,14 @@ def test_prog_seed(self): p.global_seed(10) self.assertEqual(p._seed, 10) + def test_opresult_id(self): + with paddle.pir_utils.IrGuard(): + a = paddle.static.data(name='a', shape=[4, 4], dtype='float32') + result = paddle.tanh(a) + + self.assertIsInstance(a.id, str) + self.assertIsInstance(result.id, str) + if __name__ == "__main__": unittest.main() From 3aab90132c96076be0062c04b042115014d9b6e6 Mon Sep 17 00:00:00 2001 From: kevin Date: Mon, 20 Nov 2023 10:28:16 +0800 Subject: [PATCH 06/46] [Prim][PIR] gelu forward sink (#58981) * prim gelu op sink * prim gelu op sink * update code * pir gelu sink c++ * pir gelu sink c++ * process accuracy * adapter windows * adapter windows * adapter windows --- .../decomp_interface_gen_op_list.py | 2 + paddle/fluid/primitive/composite/composite.h | 29 ++++++++++++ python/paddle/decomposition/rules.py | 26 ----------- test/legacy_test/test_activation_op.py | 4 +- test/prim/pir_prim/test_sink_decomp.py | 45 +++++++++++++++++++ 5 files changed, 79 insertions(+), 27 deletions(-) diff --git a/paddle/fluid/pir/dialect/op_generator/decomp_interface_gen_op_list.py b/paddle/fluid/pir/dialect/op_generator/decomp_interface_gen_op_list.py index af490654b91b4e..2a8b43fc09ab50 100644 --- a/paddle/fluid/pir/dialect/op_generator/decomp_interface_gen_op_list.py +++ b/paddle/fluid/pir/dialect/op_generator/decomp_interface_gen_op_list.py @@ -25,6 +25,7 @@ "relu", "softmax", "layer_norm", + "gelu", ] # come into effect in generated file op_decomp.cc @@ -36,6 +37,7 @@ "relu", "softmax", "layer_norm", + "gelu", ] diff --git a/paddle/fluid/primitive/composite/composite.h b/paddle/fluid/primitive/composite/composite.h index 75b43ccafc5fc6..9a352d74d4d3f4 100644 --- a/paddle/fluid/primitive/composite/composite.h +++ b/paddle/fluid/primitive/composite/composite.h @@ -188,6 +188,35 @@ std::tuple layer_norm_decomp( return std::make_tuple(out, mean_, variance); } +template +Tensor gelu_decomp(const Tensor& x, bool approximate) { + const double PM_2_SQRTPI = 1.12837916709551257390; /* 2/sqrt(pi) */ + const double PM_SQRT1_2 = 0.70710678118654752440; /* 1/sqrt(2) */ + + auto org_dtype = x.dtype(); + auto half = full(phi::vectorize(x.dims()), 0.5, org_dtype); + auto one = full(phi::vectorize(x.dims()), 1.0, org_dtype); + if (approximate) { + // gelu(x) = 0.5 * x * (1 + tanh(sqrt(2 / \pi) * (x + 0.044715 * x^{3}))) + auto kAlpha = + full(phi::vectorize(x.dims()), PM_2_SQRTPI * PM_SQRT1_2, org_dtype); + auto GELU_CONSTANT = full(phi::vectorize(x.dims()), 0.044715, org_dtype); + auto x_pow3 = + elementwise_pow(x, full(phi::vectorize(x.dims()), 3, org_dtype)); + auto tanh_out = tanh(kAlpha * (x + x_pow3 * GELU_CONSTANT)); + + auto res = x * half * (one + tanh_out); + return res; + } else { + // gelu(x) = 0.5 * x * (1 + erf(x / sqrt(2))) + auto M_SQRT1_2T = full(phi::vectorize(x.dims()), PM_SQRT1_2, org_dtype); + auto erf_out = one + erf(x * M_SQRT1_2T); + + auto res = x * half * erf_out; + return res; + } +} + } // namespace details } // namespace primitive diff --git a/python/paddle/decomposition/rules.py b/python/paddle/decomposition/rules.py index 12905d3101cacc..bd8a58fc680a36 100644 --- a/python/paddle/decomposition/rules.py +++ b/python/paddle/decomposition/rules.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -from paddle import _pir_ops from .primitives import * # noqa: F403 from .register import register_decomp @@ -37,31 +36,6 @@ def mean(x, axis, keepdim): return res -@register_decomp('pd_op.gelu') -def gelu(x, approximate): - """define composite rule of op gelu""" - M_SQRT1_2 = ( - 0.70710678118654752440 # /* 1/sqrt(2) */ copy from gelu-kernel.cc - ) - M_2_SQRTPI = 1.12837916709551257390 # /* 2/sqrt(pi) */ - full_shape = x.shape if len(x.shape) == 0 else [1] - one = ones(full_shape, x.dtype) - half = full(full_shape, 0.5, x.dtype) - # Todo(cz): after symbol overload, add and multiply will be replaced by "+" and "*" - if approximate: - # gelu(x) = 0.5 * x * (1 + tanh(sqrt(2 / \pi) * (x + 0.044715 * x^{3}))) - kAlpha = full(full_shape, M_2_SQRTPI * M_SQRT1_2, x.dtype) - GELU_CONSTANT = full(full_shape, 0.044715, x.dtype) - tanh_out = tanh(kAlpha * (x + GELU_CONSTANT * x * x * x)) - out = x * half * (one + tanh_out) - return out - else: - # gelu(x) = 0.5 * x * (1 + erf(x / sqrt(2))) - cdf = half * (one + _pir_ops.erf(x * full(x.shape, M_SQRT1_2, x.dtype))) - out = x * cdf - return out - - @register_decomp('pd_op.sqrt') def sqrt(x): """ diff --git a/test/legacy_test/test_activation_op.py b/test/legacy_test/test_activation_op.py index d4d120dc2696ec..40a11eec11ae7b 100644 --- a/test/legacy_test/test_activation_op.py +++ b/test/legacy_test/test_activation_op.py @@ -2691,6 +2691,8 @@ def setUp(self): self.public_python_api = paddle.nn.functional.gelu self.init_dtype() self.init_shape() + # Todo: Under float64, only this accuracy is currently supported, for further processing + self.fw_comp_rtol = 1e-7 approximate = False np.random.seed(2048) x = np.random.uniform(-1, 1, self.shape).astype(self.dtype) @@ -2713,7 +2715,7 @@ def if_enable_cinn(self): pass def test_check_output(self): - self.check_output(check_prim=True, check_pir=True, check_prim_pir=False) + self.check_output(check_prim=True, check_pir=True, check_prim_pir=True) def test_check_grad(self): if self.dtype == np.float16: diff --git a/test/prim/pir_prim/test_sink_decomp.py b/test/prim/pir_prim/test_sink_decomp.py index e9154eba60976f..b55ac33c485bdc 100644 --- a/test/prim/pir_prim/test_sink_decomp.py +++ b/test/prim/pir_prim/test_sink_decomp.py @@ -149,5 +149,50 @@ def test_relu_forward(self): np.testing.assert_equal(ref, actual) +class TestGeluSink(unittest.TestCase): + def setUp(self): + np.random.seed(2023) + self.shape_x = [8, 16, 32, 64] + self.x = np.random.random(self.shape_x).astype("float32") + self.prog = None + + def base_net(self, approximate=True, flag=None): + if flag == "forward": + core._set_prim_forward_enabled(True) + main_program = paddle.static.Program() + with paddle.static.program_guard(main_program): + x = paddle.static.data('x', self.shape_x, dtype='float32') + x.stop_gradient = False + sum_out = F.gelu(x, approximate=approximate) + [new_out] = decompose(main_program, [sum_out]) + gradients = grad(new_out, x) + + exe = paddle.static.Executor() + [fwd, dx] = exe.run( + feed={'x': self.x}, fetch_list=[new_out, gradients] + ) + + whole_ops = [op.name() for op in main_program.global_block().ops] + self.prog = main_program + if flag == "forward": + core._set_prim_forward_enabled(False) + assert 'pd_op.gelu' not in whole_ops + else: + assert 'pd_op.gelu' in whole_ops + return fwd, dx + + def test_gelu_forward_true(self): + res_ref = self.base_net(approximate=True) + res = self.base_net(approximate=True, flag="forward") + for ref, actual in zip(res_ref, res): + np.testing.assert_allclose(ref, actual, rtol=1e-6) + + def test_gelu_approximate_false(self): + res_ref = self.base_net(approximate=False) + res = self.base_net(approximate=False, flag="forward") + for ref, actual in zip(res_ref, res): + np.testing.assert_allclose(ref, actual, rtol=1e-6) + + if __name__ == "__main__": unittest.main() From cd7929afab879a179b111e0afc8e80e606d49b32 Mon Sep 17 00:00:00 2001 From: wanghuancoder Date: Mon, 20 Nov 2023 10:36:16 +0800 Subject: [PATCH 07/46] eager python c use tensor ref (#59088) --- .../eager/auto_code_generator/generator/python_c_gen.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/paddle/fluid/eager/auto_code_generator/generator/python_c_gen.py b/paddle/fluid/eager/auto_code_generator/generator/python_c_gen.py index 478dd11c3aabef..daf16f446ab12c 100644 --- a/paddle/fluid/eager/auto_code_generator/generator/python_c_gen.py +++ b/paddle/fluid/eager/auto_code_generator/generator/python_c_gen.py @@ -72,6 +72,10 @@ def FindParsingFunctionFromAttributeType(atype): " auto {} = {}(\"{}\", \"{}\", args, {}, {});\n" ) +PARSE_PYTHON_C_TENSOR_REF_TEMPLATE = ( + " auto& {} = {}(\"{}\", \"{}\", args, {}, {});\n" +) + CONVERT_TO_DISTTENSOR_AND_PARSE_PYTHON_C_TENSORS_TEMPLATE = ( " {} = {}(\"{}\", \"{}\", args, {}, {}, mesh);\n" ) @@ -390,7 +394,7 @@ def GeneratePythonCFunction(self): input_single_tensor_names + ", " + name ) get_eager_tensor_str += ( - PARSE_PYTHON_C_TENSORS_TEMPLATE.format( + PARSE_PYTHON_C_TENSOR_REF_TEMPLATE.format( name, "GetTensorFromArgs", forward_api_name, From c56dd398a1c5173ba3085c90ee32d71417b54c79 Mon Sep 17 00:00:00 2001 From: xiongkun Date: Mon, 20 Nov 2023 10:53:06 +0800 Subject: [PATCH 08/46] [SOT] add mechanism for binary-tracker: Part I for dynamic shape in sot. (#59035) --- .../executor/pycode_generator.py | 21 +++ .../sot/opcode_translator/executor/tracker.py | 49 +++++++ test/sot/test_binary_operator_tracker.py | 130 ++++++++++++++++++ 3 files changed, 200 insertions(+) create mode 100644 test/sot/test_binary_operator_tracker.py diff --git a/python/paddle/jit/sot/opcode_translator/executor/pycode_generator.py b/python/paddle/jit/sot/opcode_translator/executor/pycode_generator.py index 29764afdca4eb7..07038d14b46fad 100644 --- a/python/paddle/jit/sot/opcode_translator/executor/pycode_generator.py +++ b/python/paddle/jit/sot/opcode_translator/executor/pycode_generator.py @@ -1026,6 +1026,27 @@ def gen_return(self): def gen_get_iter(self): self._add_instr("GET_ITER") + def gen_operator_only(self, op_name): + """ + only generator operator instruction, do nothing for + operands. + """ + self._add_instr(op_name) + + def gen_operator(self, op_name): + """ + only generator operator instruction, do nothing for + operands. + """ + self._add_instr(op_name) + + def gen_compare(self, cmp_op): + """ + only generator operator instruction, do nothing for + operands. + """ + self._add_instr("COMPARE_OP", cmp_op) + def _add_instr(self, *args, **kwargs): instr = gen_instr(*args, **kwargs) self._instructions.append(instr) diff --git a/python/paddle/jit/sot/opcode_translator/executor/tracker.py b/python/paddle/jit/sot/opcode_translator/executor/tracker.py index c085e14b5b3824..fd7168f4e5957f 100644 --- a/python/paddle/jit/sot/opcode_translator/executor/tracker.py +++ b/python/paddle/jit/sot/opcode_translator/executor/tracker.py @@ -15,6 +15,7 @@ from __future__ import annotations import builtins +import dis import sys from typing import TYPE_CHECKING @@ -249,6 +250,54 @@ def need_guard(self) -> bool: return False +class BinaryOperatorTracker(Tracker): + def __init__( + self, operator: str, operands: list[VariableBase], addition=None + ): + """ + addition is for the case that the operator is "COMPARE_OP", which represents the dis.cmp_op's index. + """ + super().__init__(operands, False) + assert len(operands) == 2, "Currently only support binary operator." + self.operands = operands + self.operator = operator + self.addition = addition + + def gen_instructions(self, codegen: PyCodeGen): + for operand in self.operands: + operand.tracker.gen_instructions(codegen) + self.gen_operator_instr(codegen) + + def gen_operator_instr(self, codegen: PyCodeGen): + if self.operator == "COMPARE_OP": + codegen.gen_compare(self.addition) + else: + codegen.gen_operator(self.operator) + + def get_operator_symbol(self): + if self.operator == "COMPARE_OP": + return dis.cmp_op[self.addition] + return { + "BINARY_ADD": "+", + "BINARY_SUBTRACT": "-", + "BINARY_MUL": "*", + "BINARY_POWER": "**", + }[self.operator] + + def trace_value_from_frame(self): + sub_exprs = [x.tracker.trace_value_from_frame() for x in self.operands] + sub_frees = [x.free_vars for x in sub_exprs] + expr = f"({{}} {self.get_operator_symbol()} {{}})" + return StringifyExpression( + expr, + list(sub_exprs), + union_free_vars(*list(sub_frees)), + ) + + def __repr__(self) -> str: + return f"BinaryOperatorTracker(operator={self.operator})" + + class GetAttrTracker(Tracker): """ GetAttrTracker is a subclass of Tracker that specifically tracks the attribute access of an variable. diff --git a/test/sot/test_binary_operator_tracker.py b/test/sot/test_binary_operator_tracker.py new file mode 100644 index 00000000000000..be74e2ebfdd4b4 --- /dev/null +++ b/test/sot/test_binary_operator_tracker.py @@ -0,0 +1,130 @@ +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# GET_ITER (new) +# FOR_ITER (new) + +from __future__ import annotations + +import os + +os.environ['MIN_GRAPH_SIZE'] = '-1' +import operator +import unittest + +from test_case_base import ( + TestCaseBase, + test_instruction_translator_cache_context, +) + +import paddle +from paddle.jit.sot.opcode_translator.executor.dispatcher import ( + Dispatcher, + Parameter, + Pattern, +) +from paddle.jit.sot.opcode_translator.executor.tracker import ( + BinaryOperatorTracker, +) +from paddle.jit.sot.opcode_translator.executor.variables import ConstantVariable + + +def register_dispatch(fn, parameters, handler): + """ + Registering function signature. + + Args: + fn: The function to be registered. + parameters: The parameters of the function to be registered. + handler: The handler function. + """ + _parameters = tuple( + Parameter.from_str(parameter) + if isinstance(parameter, str) + else parameter + for parameter in parameters + ) + if fn not in Dispatcher.handlers: + Dispatcher.handlers[fn] = [] + Dispatcher.handlers[fn].insert(0, (Pattern(*_parameters), handler)) + + +def operator_gt_dis_func(left, right): + return ConstantVariable( + operator.gt(left.get_py_value(), right.get_py_value()), + graph=left.graph, + tracker=BinaryOperatorTracker("COMPARE_OP", [left, right], 4), + ) + + +def operator_add_dis_func(left, right): + return ConstantVariable( + operator.gt(left.get_py_value(), right.get_py_value()), + graph=left.graph, + tracker=BinaryOperatorTracker("BINARY_ADD", [left, right]), + ) + + +class TestBinaryOperatorTracker(TestCaseBase): + def test_case_compare_op(self): + def func(x, y): + if x > 0: + return y + 1 + return y + 2 + + register_dispatch( + operator.gt, + ("ConstantVariable", "ConstantVariable"), + operator_gt_dis_func, + ) + + y = paddle.randn((2, 2)) + with test_instruction_translator_cache_context() as ctx: + self.assert_results(func, 12, y) + self.assertEqual(ctx.translate_count, 1) + self.assert_results(func, 10, y) + self.assertEqual(ctx.translate_count, 1) + self.assert_results(func, -12, y) + self.assertEqual(ctx.translate_count, 2) + + def test_case_compare_op_2(self): + def func(x, y): + if x + 2 > 0: + return y + 1 + return y + 2 + + register_dispatch( + operator.gt, + ("ConstantVariable", "ConstantVariable"), + operator_gt_dis_func, + ) + + register_dispatch( + operator.add, + ("ConstantVariable", "ConstantVariable"), + operator_add_dis_func, + ) + + y = paddle.randn((2, 2)) + with test_instruction_translator_cache_context() as ctx: + self.assert_results(func, 12, y) + self.assertEqual(ctx.translate_count, 1) + self.assert_results(func, 10, y) + self.assertEqual(ctx.translate_count, 1) + self.assert_results(func, -12, y) + self.assertEqual(ctx.translate_count, 2) + + +if __name__ == "__main__": + unittest.main() From e05d3e91f8df09afb8871dc71e115cb150cd3c25 Mon Sep 17 00:00:00 2001 From: xiaoguoguo626807 <100397923+xiaoguoguo626807@users.noreply.github.com> Date: Mon, 20 Nov 2023 10:57:09 +0800 Subject: [PATCH 09/46] =?UTF-8?q?=E3=80=90pir=E3=80=91modify=20ir=20Backwa?= =?UTF-8?q?rd=20for=20prune=20(#59100)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * tmp * modify ci bug * [PIR]Migrate maximum into pir * Polish code * add ir_grad of static_gradient * add test * modify backward * modify * modify segment --------- Co-authored-by: 0x45f --- python/paddle/autograd/ir_backward.py | 88 +++++++++++++++++---------- test/ir/pir/test_ir_backward.py | 2 +- test/legacy_test/test_segment_ops.py | 8 ++- 3 files changed, 63 insertions(+), 35 deletions(-) diff --git a/python/paddle/autograd/ir_backward.py b/python/paddle/autograd/ir_backward.py index 9ad445d62ff1c6..8e112012599b81 100644 --- a/python/paddle/autograd/ir_backward.py +++ b/python/paddle/autograd/ir_backward.py @@ -58,9 +58,10 @@ def update_no_grad_set_by_stopgradient(block, no_grad_set): no_grad_set.add(value) -def update_bwdop_structure(backward_ops, op_to_opgrad_list, grad_op): - backward_ops.append(grad_op) - op_to_opgrad_list.append(grad_op) +def update_bwdop_structure(backward_ops, op_to_opgrad_list, grad_op_list): + for grad_op in grad_op_list: + backward_ops.append(grad_op) + op_to_opgrad_list.append(grad_op) def prepare_grad_outputs(grad_outputs, outputs, state): @@ -87,18 +88,19 @@ def prepare_grad_outputs(grad_outputs, outputs, state): for i, grad in enumerate(grad_outputs): output = outputs[i] # fwd : op1 -> op2 -> op3 -> output - # bwd : op1G <- op2G <- op3G <- outputG <- fillop/feedop + # bwd : op1G <- op2G <- op3G <- outputG <- full_likeop/feedop if grad is None: output_grad = paddle.full_like( output, 1.0, dtype=output.dtype, ) - fillop = output_grad.get_defining_op() + full_likeop = output_grad.get_defining_op() + fullop = full_likeop.operand_source(1).get_defining_op() update_bwdop_structure( backward_ops, state.op_to_opgrad[output.get_defining_op()], - fillop, + [full_likeop, fullop], ) state.value_to_valuegrad[output] = [[output_grad]] else: @@ -116,7 +118,7 @@ def prepare_grad_outputs(grad_outputs, outputs, state): update_bwdop_structure( backward_ops, state.op_to_opgrad[output.get_defining_op()], - feedop, + [feedop], ) state.value_to_valuegrad[output] = [[grad]] @@ -138,12 +140,13 @@ def prepare_grad_outputs(grad_outputs, outputs, state): 0.0, opresult.dtype, ) - fillop = grad_value.get_defining_op() + full_likeop = grad_value.get_defining_op() + fullop = full_likeop.operand_source(1).get_defining_op() update_bwdop_structure( backward_ops, state.op_to_opgrad[opresult.get_defining_op()], - fillop, + [full_likeop, fullop], ) state.value_to_valuegrad[opresult] = [[grad_value]] @@ -383,11 +386,9 @@ def make_output_with_output_grad(op): combineop = bwd_block.ops[len(bwd_block.ops) - 2] sumop = bwd_block.ops[len(bwd_block.ops) - 1] update_bwdop_structure( - backward_ops, state.op_to_opgrad[op], combineop - ) - update_bwdop_structure( - backward_ops, state.op_to_opgrad[op], sumop + backward_ops, state.op_to_opgrad[op], [combineop, sumop] ) + state.value_to_valuegrad[value] = [[sumop.result(0)]] state.value_to_sumvaluegrad[value] = state.value_to_valuegrad[ value @@ -426,10 +427,13 @@ def make_output_with_output_grad(op): 0.0, dtype=value.dtype, ) - fillop = grad_value.get_defining_op() + full_likeop = grad_value.get_defining_op() + fullop = full_likeop.operand_source(1).get_defining_op() update_bwdop_structure( - backward_ops, state.op_to_opgrad[op], fillop + backward_ops, + state.op_to_opgrad[op], + [full_likeop, fullop], ) zero_flag[i] = True @@ -548,10 +552,12 @@ def update_input_grad_map(op, input_grads): after_ops_num = len(bwd_block.ops) # update grad_op structure - for i in range(before_ops_num, after_ops_num): - update_bwdop_structure( - backward_ops, state.op_to_opgrad[op], bwd_block.ops[i] - ) + bwd_ops = [ + bwd_block.ops[i] for i in range(before_ops_num, after_ops_num) + ] + update_bwdop_structure( + backward_ops, state.op_to_opgrad[op], bwd_ops + ) # update input_grad map update_input_grad_map(op, input_grads) @@ -570,10 +576,9 @@ def update_input_grad_map(op, input_grads): combineop = bwd_block.ops[len(bwd_block.ops) - 2] sumop = bwd_block.ops[len(bwd_block.ops) - 1] update_bwdop_structure( - backward_ops, state.op_to_opgrad[op], combineop - ) - update_bwdop_structure( - backward_ops, state.op_to_opgrad[op], sumop + backward_ops, + state.op_to_opgrad[op], + [combineop, sumop], ) state.value_to_valuegrad[value] = [[sumop.result(0)]] state.value_to_sumvaluegrad[ @@ -585,20 +590,35 @@ def update_input_grad_map(op, input_grads): state.op_to_opgrad[op] = [] -def create_backward_prune_set(inputs, outputs, no_grad_set, state): - outputs_set = set() +def prepare_backward_prune_set(inputs, outputs): + outputs_fwd_set = set() for input_ in inputs: if not input_.use_empty(): for item in input_.first_use().owner().operands_source(): - if state.value_to_valuegrad[item] != []: - outputs_set.add(state.value_to_valuegrad[item][0][0]) + outputs_fwd_set.add(item) else: logging.warning("input privided by inputs has no use") - inputs_set = set() + inputs_fwd_set = set() for output in outputs: - if state.value_to_valuegrad[output] != []: - inputs_set.add(state.value_to_valuegrad[output][0][0]) + inputs_fwd_set.add(output) + + return outputs_fwd_set, inputs_fwd_set + + +def create_backward_prune_set( + outputs_fwd_set, inputs_fwd_set, no_grad_set, state +): + outputs_set = set() + for item in outputs_fwd_set: + if state.value_to_valuegrad[item] != []: + outputs_set.add(state.value_to_valuegrad[item][0][0]) + + inputs_set = set() + for item in inputs_fwd_set: + if state.value_to_valuegrad[item] != []: + inputs_set.add(state.value_to_valuegrad[item][0][0]) + inputs_set_tmp = set() for out_grad in inputs_set: if not out_grad.use_empty(): @@ -660,13 +680,19 @@ def calc_gradient_helper(outputs, inputs, grad_outputs, no_grad_set): block, effective_forward_ops, no_grad_set, inputs, complete_outputs ) + outputs_fwd_set, inputs_fwd_set = prepare_backward_prune_set( + inputs, complete_outputs + ) + append_backward_ops( block, block, effective_forward_ops, no_grad_set, backward_ops, state ) + # now value_to_valuegrad should be value <-> value (add sum op for the same values's gradvalue) outputs_set, inputs_set, no_gradvar_set = create_backward_prune_set( - inputs, complete_outputs, no_grad_set, state + outputs_fwd_set, inputs_fwd_set, no_grad_set, state ) + _, remove_ops = prune_ops( backward_ops, inputs_set, outputs_set, no_gradvar_set ) diff --git a/test/ir/pir/test_ir_backward.py b/test/ir/pir/test_ir_backward.py index b429813e3ec6f6..31e1254030183d 100644 --- a/test/ir/pir/test_ir_backward.py +++ b/test/ir/pir/test_ir_backward.py @@ -101,7 +101,7 @@ def test_no_grad_set(self): out = paddle.mean(tanh_out) input_grad = grad(out, input, no_grad_vars=[input]) self.assertEqual( - pir_program.global_block().ops[-1].name(), "pd_op.full" + pir_program.global_block().ops[-1].name(), "pd_op.mean" ) def test_split(self): diff --git a/test/legacy_test/test_segment_ops.py b/test/legacy_test/test_segment_ops.py index a78db61b6e399e..8278cd984d1d61 100644 --- a/test/legacy_test/test_segment_ops.py +++ b/test/legacy_test/test_segment_ops.py @@ -127,7 +127,7 @@ def test_check_output(self): self.check_output(check_pir=True) def test_check_grad(self): - self.check_grad(["X"], "Out") + self.check_grad(["X"], "Out", check_pir=True) def convert_bf16(self): if self.dtype == np.uint16: @@ -277,7 +277,7 @@ def test_check_output(self): self.check_output_with_place(self.place, check_pir=True) def test_check_grad(self): - self.check_grad_with_place(self.place, ["X"], "Out") + self.check_grad_with_place(self.place, ["X"], "Out", check_pir=True) @unittest.skipIf( @@ -300,6 +300,7 @@ def test_check_grad(self): ["X"], "Out", user_defined_grads=[self.gradient], + check_pir=True, ) @@ -323,6 +324,7 @@ def test_check_grad(self): ["X"], "Out", user_defined_grads=[self.gradient], + check_pir=True, ) @@ -341,7 +343,7 @@ def test_check_output(self): self.check_output_with_place(self.place, check_pir=True) def test_check_grad(self): - self.check_grad_with_place(self.place, ["X"], "Out") + self.check_grad_with_place(self.place, ["X"], "Out", check_pir=True) class API_SegmentOpsTest(unittest.TestCase): From fd9105288624cd2f2684bb4cd926ae91cf966618 Mon Sep 17 00:00:00 2001 From: HongyuJia Date: Mon, 20 Nov 2023 10:57:27 +0800 Subject: [PATCH 10/46] [CINN] Fix ir_printer when print Dim (#59112) * Fix ir_printer when print Dim * Remove the change_line code --- paddle/cinn/ir/ir_printer.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/paddle/cinn/ir/ir_printer.cc b/paddle/cinn/ir/ir_printer.cc index f0e72354eff90d..b46a8028798370 100644 --- a/paddle/cinn/ir/ir_printer.cc +++ b/paddle/cinn/ir/ir_printer.cc @@ -613,8 +613,8 @@ void IrPrinter::Visit(const _Dim_ *x) { str_ += ", sym_name: "; str_ += x->GetSymbolName(); str_ += ", dim_size: "; - str_ += x->GetRealDimSize(); - str_ += ")\n"; + str_ += std::to_string(x->GetRealDimSize()); + str_ += ")"; } void IrPrinter::Visit(const IntrinsicOp *x) { From abac31fd95060c5d8b5a1808774a984c194e04e6 Mon Sep 17 00:00:00 2001 From: Zhan Rongrui <46243324+zrr1999@users.noreply.github.com> Date: Mon, 20 Nov 2023 11:22:41 +0800 Subject: [PATCH 11/46] Migrate print into pir (#58780) --- .../pir/dialect/op_generator/ops_api_gen.py | 2 +- python/paddle/static/nn/control_flow.py | 19 ++- test/legacy_test/test_index_put_op.py | 6 +- test/legacy_test/test_print_op.py | 111 +++++++++++------- 4 files changed, 94 insertions(+), 44 deletions(-) diff --git a/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py b/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py index 1b78209d2223da..1075065cd07551 100644 --- a/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py +++ b/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py @@ -92,6 +92,7 @@ 'fc', 'self_dp_attention', 'get_tensor_from_selected_rows', + 'print', ] NO_NEED_GEN_STATIC_ONLY_APIS = [ @@ -114,7 +115,6 @@ 'fused_scale_bias_relu_conv_bn', 'fused_scale_bias_add_relu', 'memcpy', - 'print', 'recv_v2', 'rnn_', 'seed', diff --git a/python/paddle/static/nn/control_flow.py b/python/paddle/static/nn/control_flow.py index 32b4f4407be233..0f00cf4915295c 100644 --- a/python/paddle/static/nn/control_flow.py +++ b/python/paddle/static/nn/control_flow.py @@ -16,6 +16,7 @@ from functools import partial, reduce import paddle +from paddle import _C_ops from paddle.base import core from paddle.base.backward import _infer_var_data_type_shape_ from paddle.base.framework import ( @@ -1745,8 +1746,24 @@ def Print( ['uint16', 'float16', 'float32', 'float64', 'int32', 'int64', 'bool'], 'paddle.static.Print', ) + message = message or "" + helper = LayerHelper('print', **locals()) + + if in_pir_mode(): + return _C_ops.print( + input, + first_n, + message, + summarize, + print_tensor_name, + print_tensor_type, + print_tensor_shape, + print_tensor_layout, + print_tensor_lod, + print_phase.upper(), + True, + ) - helper = LayerHelper('print' + "_" + input.name, **locals()) output = helper.create_variable_for_type_inference(input.dtype) helper.append_op( type='print', diff --git a/test/legacy_test/test_index_put_op.py b/test/legacy_test/test_index_put_op.py index 3d988462194cca..ca8b5389f8b37e 100644 --- a/test/legacy_test/test_index_put_op.py +++ b/test/legacy_test/test_index_put_op.py @@ -18,6 +18,7 @@ import numpy as np import paddle +from paddle.pir_utils import test_with_pir_api def compute_index_put_ref(x_np, indices_np, value_np, accumulate=False): @@ -143,7 +144,7 @@ def test_dygraph_forward(self): ) np.testing.assert_allclose(ref_res, pd_res.numpy(), atol=1e-7) - # @test_with_pir_api + @test_with_pir_api def test_static_forward(self): paddle.enable_static() for place in self.place: @@ -626,6 +627,7 @@ def setPlace(self): if paddle.is_compiled_with_cuda(): self.place.append('gpu') + @test_with_pir_api def test_dygraph_forward(self): paddle.disable_static() for place in self.place: @@ -934,7 +936,7 @@ def test_backward_all_false_bool_indice(self): atol=1e-7, ) - # @test_with_pir_api + @test_with_pir_api def test_backward_in_static(self): paddle.enable_static() exe = paddle.static.Executor() diff --git a/test/legacy_test/test_print_op.py b/test/legacy_test/test_print_op.py index c4390d76bb9ffd..95c1dd420626d7 100755 --- a/test/legacy_test/test_print_op.py +++ b/test/legacy_test/test_print_op.py @@ -19,8 +19,10 @@ import paddle from paddle import base +from paddle.autograd.ir_backward import grad from paddle.base import core -from paddle.base.framework import switch_main_program +from paddle.framework import in_dynamic_or_pir_mode +from paddle.pir_utils import test_with_pir_api from paddle.static import Program, program_guard paddle.enable_static() @@ -39,54 +41,81 @@ def build_network(self, only_forward, **kargs): x.stop_gradient = False paddle.static.Print(input=x, **kargs) loss = paddle.mean(x) - paddle.static.append_backward(loss=loss) + + if in_dynamic_or_pir_mode(): + dx = grad(loss, [x]) + else: + paddle.static.append_backward(loss=loss) return loss + @test_with_pir_api def test_forward(self): - switch_main_program(Program()) - printed = self.build_network(True, print_phase='forward') - exe = paddle.static.Executor(self.place) - outs = exe.run( - feed={'x': self.x_tensor}, fetch_list=[printed], return_numpy=False - ) + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): + printed = self.build_network(True, print_phase='forward') + exe = paddle.static.Executor(self.place) + outs = exe.run( + feed={'x': self.x_tensor}, + fetch_list=[printed], + return_numpy=False, + ) + @test_with_pir_api def test_backward(self): - switch_main_program(Program()) - loss = self.build_network(False, print_phase='backward') - exe = paddle.static.Executor(self.place) - outs = exe.run( - feed={'x': self.x_tensor}, fetch_list=[loss], return_numpy=False - ) + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): + loss = self.build_network(False, print_phase='backward') + exe = paddle.static.Executor(self.place) + outs = exe.run( + feed={'x': self.x_tensor}, fetch_list=[loss], return_numpy=False + ) + @test_with_pir_api def test_all_parameters(self): - x = paddle.static.data('x', shape=[-1, 3], dtype='float32', lod_level=1) - x.stop_gradient = False - - for print_tensor_name in [True, False]: - for print_tensor_type in [True, False]: - for print_tensor_shape in [True, False]: - for print_tensor_lod in [True, False]: - paddle.static.Print( - input=x, - print_tensor_name=print_tensor_name, - print_tensor_type=print_tensor_type, - print_tensor_shape=print_tensor_shape, - print_tensor_lod=print_tensor_lod, - ) - loss = paddle.mean(x) - paddle.static.append_backward(loss=loss) - exe = paddle.static.Executor(self.place) - outs = exe.run( - feed={'x': self.x_tensor}, fetch_list=[loss], return_numpy=False - ) + prog = paddle.static.Program() + with paddle.static.program_guard(prog, paddle.static.Program()): + x = paddle.static.data( + 'x', shape=[-1, 3], dtype='float32', lod_level=1 + ) + x.stop_gradient = False + + for print_tensor_name in [True, False]: + for print_tensor_type in [True, False]: + for print_tensor_shape in [True, False]: + for print_tensor_lod in [True, False]: + paddle.static.Print( + input=x, + print_tensor_name=print_tensor_name, + print_tensor_type=print_tensor_type, + print_tensor_shape=print_tensor_shape, + print_tensor_lod=print_tensor_lod, + ) + loss = paddle.mean(x) + if in_dynamic_or_pir_mode(): + dx = grad(loss, [x]) + else: + paddle.static.append_backward(loss=loss) + exe = paddle.static.Executor(self.place) + outs = exe.run( + feed={'x': self.x_tensor}, fetch_list=[loss], return_numpy=False + ) + @test_with_pir_api def test_no_summarize(self): - switch_main_program(Program()) - printed = self.build_network(True, summarize=-1, print_phase='forward') - exe = paddle.static.Executor(self.place) - outs = exe.run( - feed={'x': self.x_tensor}, fetch_list=[printed], return_numpy=False - ) + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): + printed = self.build_network( + True, summarize=-1, print_phase='forward' + ) + exe = paddle.static.Executor(self.place) + outs = exe.run( + feed={'x': self.x_tensor}, + fetch_list=[printed], + return_numpy=False, + ) class TestPrintOpError(unittest.TestCase): @@ -137,6 +166,8 @@ def check_backward(self, use_cuda): feed_dict = {"image": img, "label": label} exe.run(binary, feed_dict) + # fc is not supported in pir + # @test_with_pir_api def test_fw_bw(self): if paddle.is_compiled_with_cuda(): self.check_backward(use_cuda=True) From 743f882cfc5a6745bf63dfe6a90deb029fd508ad Mon Sep 17 00:00:00 2001 From: ooo oo <106524776+ooooo-create@users.noreply.github.com> Date: Mon, 20 Nov 2023 11:52:33 +0800 Subject: [PATCH 12/46] change .. code-block:: text to .. code-block:: python in example code, in python\paddle\distributed\fleet\fleet.py (#59116) --- python/paddle/distributed/fleet/fleet.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/distributed/fleet/fleet.py b/python/paddle/distributed/fleet/fleet.py index f18f7aeb068761..28b8f8dd353458 100755 --- a/python/paddle/distributed/fleet/fleet.py +++ b/python/paddle/distributed/fleet/fleet.py @@ -957,7 +957,7 @@ def save_persistables(self, executor, dirname, main_program=None, mode=0): Examples: - .. code-block:: text + .. code-block:: python >>> import paddle >>> paddle.enable_static() From e2517d1d09a8096001ab74c129765421d84e5836 Mon Sep 17 00:00:00 2001 From: Dingyuan Wang Date: Mon, 20 Nov 2023 12:02:25 +0800 Subject: [PATCH 13/46] Fix building on GCC 13 and Linux 6.0+ (#59078) * fix header for GCC 13, fix gloo for linux 6.0+ * fix code style --- cmake/external/gloo.cmake | 11 +++++++++++ .../platform/profiler/chrometracing_logger.h | 1 + paddle/phi/common/place.h | 1 + paddle/phi/core/os_info.h | 1 + paddle/utils/string/string_helper.h | 17 +++++++++-------- patches/gloo/linux.cc.patch | 13 +++++++++++++ 6 files changed, 36 insertions(+), 8 deletions(-) create mode 100644 patches/gloo/linux.cc.patch diff --git a/cmake/external/gloo.cmake b/cmake/external/gloo.cmake index 64c5acf70f99f0..529f72b662e3e2 100755 --- a/cmake/external/gloo.cmake +++ b/cmake/external/gloo.cmake @@ -61,6 +61,17 @@ if(CMAKE_COMPILER_IS_GNUCC) ${SOURCE_DIR}/gloo/ < ${types_header}) endif() endif() + +file(TO_NATIVE_PATH ${PADDLE_SOURCE_DIR}/patches/gloo/linux.cc.patch + linux_cc_ethtool) +if(GLOO_PATCH_COMMAND STREQUAL "") + set(GLOO_PATCH_COMMAND git checkout -- . && git checkout ${GLOO_TAG} && patch + -Nd ${SOURCE_DIR}/gloo/common/ < ${linux_cc_ethtool}) +else() + set(GLOO_PATCH_COMMAND ${GLOO_PATCH_COMMAND} && patch -Nd + ${SOURCE_DIR}/gloo/common/ < ${linux_cc_ethtool}) +endif() + include_directories(${GLOO_INCLUDE_DIR}) ExternalProject_Add( diff --git a/paddle/fluid/platform/profiler/chrometracing_logger.h b/paddle/fluid/platform/profiler/chrometracing_logger.h index 7f9bec1c32a534..37323d1450bf2d 100644 --- a/paddle/fluid/platform/profiler/chrometracing_logger.h +++ b/paddle/fluid/platform/profiler/chrometracing_logger.h @@ -13,6 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. */ #pragma once +#include #include #include #include diff --git a/paddle/phi/common/place.h b/paddle/phi/common/place.h index da2e4e489e823d..03072468f62e20 100644 --- a/paddle/phi/common/place.h +++ b/paddle/phi/common/place.h @@ -14,6 +14,7 @@ limitations under the License. */ #pragma once +#include #include #include diff --git a/paddle/phi/core/os_info.h b/paddle/phi/core/os_info.h index 829e944eca8414..eb93590669da3f 100644 --- a/paddle/phi/core/os_info.h +++ b/paddle/phi/core/os_info.h @@ -14,6 +14,7 @@ limitations under the License. */ #pragma once +#include #include #include #ifdef _POSIX_C_SOURCE diff --git a/paddle/utils/string/string_helper.h b/paddle/utils/string/string_helper.h index 004abfd9da2e7a..01e0cb0b4eb858 100644 --- a/paddle/utils/string/string_helper.h +++ b/paddle/utils/string/string_helper.h @@ -19,6 +19,7 @@ #include #include +#include #include #include #include @@ -102,12 +103,12 @@ inline int str_to_float(const char* str, float* v) { return index; } -inline float* str_to_float(std::string& str) { - return (float*)const_cast(str.c_str()); +inline float* str_to_float(const std::string& str) { + return reinterpret_cast(const_cast(str.c_str())); } inline float* str_to_float(const char* str) { - return (float*)const_cast(str); + return reinterpret_cast(const_cast(str)); } // checks whether the test string is a suffix of the input string. @@ -247,7 +248,7 @@ struct str_ptr_stream { char* ptr = NULL; char* end = NULL; str_ptr_stream() {} - str_ptr_stream(const str_ptr& p) { reset(p.ptr, p.len); } + explicit str_ptr_stream(const str_ptr& p) { reset(p.ptr, p.len); } void reset(const str_ptr& p) { reset(p.ptr, p.len); } void reset(const char* p, size_t len) { ptr = const_cast(p); @@ -316,7 +317,7 @@ inline int split_string_ptr(const char* str, ++p; continue; } - values->emplace_back(last, (size_t)(p - last)); + values->emplace_back(last, static_cast(p - last)); ++num; ++p; // skip continue delim @@ -326,7 +327,7 @@ inline int split_string_ptr(const char* str, last = p; } if (p > last) { - values->emplace_back(last, (size_t)(p - last)); + values->emplace_back(last, static_cast(p - last)); ++num; } return num; @@ -350,7 +351,7 @@ inline int split_string_ptr(const char* str, ++p; continue; } - values->emplace_back(last, (size_t)(p - last)); + values->emplace_back(last, static_cast(p - last)); ++num; ++p; if (num >= max_num) { @@ -363,7 +364,7 @@ inline int split_string_ptr(const char* str, last = p; } if (p > last) { - values->emplace_back(last, (size_t)(p - last)); + values->emplace_back(last, static_cast(p - last)); ++num; } return num; diff --git a/patches/gloo/linux.cc.patch b/patches/gloo/linux.cc.patch new file mode 100644 index 00000000000000..d594eeae24e961 --- /dev/null +++ b/patches/gloo/linux.cc.patch @@ -0,0 +1,13 @@ +diff --git a/gloo/common/linux.cc b/gloo/common/linux.cc +index a3726da..9a42a12 100644 +--- a/linux.cc ++++ b/linux.cc +@@ -188,8 +188,8 @@ static int getInterfaceSpeedGLinkSettings(int sock, struct ifreq* ifr) { + #if LINUX_VERSION_CODE >= KERNEL_VERSION(4,6,0) + constexpr auto link_mode_data_nwords = 3 * 127; + struct { +- struct ethtool_link_settings req; + __u32 link_mode_data[link_mode_data_nwords]; ++ struct ethtool_link_settings req; + } ecmd; + int rv; From cbafa02ad434ec4e7803d81192572b03d8edb262 Mon Sep 17 00:00:00 2001 From: HandSomeLEEw <48877749+HandSomeLEEw@users.noreply.github.com> Date: Mon, 20 Nov 2023 12:15:37 +0800 Subject: [PATCH 14/46] add MergedAdamKernel and test for MergedAdamKernel and fix adam caculation process in test (#58982) --- paddle/phi/backends/xpu/xpu2_op_list.cc | 1 + paddle/phi/kernels/xpu/adam_kernel.cc | 231 ++++++++++++++++++++ test/xpu/test_adam_op_xpu.py | 4 +- test/xpu/test_merged_adam_op_xpu.py | 279 ++++++++++++++++++++++++ 4 files changed, 514 insertions(+), 1 deletion(-) create mode 100644 test/xpu/test_merged_adam_op_xpu.py diff --git a/paddle/phi/backends/xpu/xpu2_op_list.cc b/paddle/phi/backends/xpu/xpu2_op_list.cc index df76d2ac366755..bb2440101efbed 100644 --- a/paddle/phi/backends/xpu/xpu2_op_list.cc +++ b/paddle/phi/backends/xpu/xpu2_op_list.cc @@ -582,6 +582,7 @@ XPUOpMap& get_kl2_ops() { {"mean_grad", XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})}, {"mean", XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})}, + {"merged_adam", XPUKernelSet({phi::DataType::FLOAT32})}, {"merged_momentum", XPUKernelSet({phi::DataType::FLOAT32, phi::DataType::FLOAT16})}, {"mish_grad", XPUKernelSet({phi::DataType::FLOAT32})}, diff --git a/paddle/phi/kernels/xpu/adam_kernel.cc b/paddle/phi/kernels/xpu/adam_kernel.cc index f2e9fccc58da37..c3fd153ebd3c0e 100644 --- a/paddle/phi/kernels/xpu/adam_kernel.cc +++ b/paddle/phi/kernels/xpu/adam_kernel.cc @@ -240,6 +240,229 @@ void AdamDenseKernel(const Context& dev_ctx, funcs::FreeData(moment2, mom2_ptr); funcs::FreeData(learning_rate, lr_ptr); } + +template +void MergedAdamKernel( + const Context& dev_ctx, + const std::vector& param, + const std::vector& grad, + const std::vector& learning_rate, + const std::vector& moment1, + const std::vector& moment2, + const std::vector& beta1_pow, + const std::vector& beta2_pow, + const paddle::optional>& master_param, + const Scalar& beta1, + const Scalar& beta2, + const Scalar& epsilon, + bool multi_precision, + bool use_global_beta_pow, + std::vector param_out, + std::vector moment1_out, + std::vector moment2_out, + std::vector beta1_pow_out, + std::vector beta2_pow_out, + std::vector master_param_out) { + VLOG(4) << "use_global_beta_pow:" << use_global_beta_pow; + + auto beta1_ = beta1.to(); + auto beta2_ = beta2.to(); + auto epsilon_ = epsilon.to(); + int64_t step_ = 0; + int64_t mode_ = 2; + int64_t bias_correction_ = 1; + float weight_decay_ = 0.0; + + DenseTensor lr_host; + lr_host.Resize(learning_rate[0]->dims()); + dev_ctx.template HostAlloc(&lr_host); + phi::Copy(dev_ctx, *learning_rate[0], CPUPlace(), false, &lr_host); + float lr_ = *(lr_host.template data()); + + float beta1_pow_data; + if (beta1_pow[0]->place() == CPUPlace()) { + beta1_pow_data = *(beta1_pow[0]->data()); + } else { + DenseTensor beta1_pow_host; + beta1_pow_host.Resize(beta1_pow[0]->dims()); + dev_ctx.template HostAlloc(&beta1_pow_host); + phi::Copy(dev_ctx, *beta1_pow[0], CPUPlace(), false, &beta1_pow_host); + beta1_pow_data = *(beta1_pow_host.template data()); + } + + float beta2_pow_data; + if (beta2_pow[0]->place() == CPUPlace()) { + beta2_pow_data = *(beta2_pow[0]->data()); + } else { + DenseTensor beta2_pow_host; + beta2_pow_host.Resize(beta2_pow[0]->dims()); + dev_ctx.template HostAlloc(&beta2_pow_host); + phi::Copy(dev_ctx, *beta2_pow[0], CPUPlace(), false, &beta2_pow_host); + beta2_pow_data = *(beta2_pow_host.template data()); + } + + int param_num = param.size(); + PADDLE_ENFORCE_EQ(param_num, + param_out.size(), + errors::InvalidArgument( + "The size of Output(ParamOut) must be equal to " + "Input(Param), but got the size of Output(ParamOut) " + "is %d, the size of Input(Param) is %d.", + param_out.size(), + param_num)); + PADDLE_ENFORCE_EQ( + param_num, + moment1_out.size(), + errors::InvalidArgument( + "The size of Input(Moment1) must be equal to Input(Param), but got " + "the size of Input(Moment1) is %d, the size of Input(Param) is %d.", + moment1.size(), + param_num)); + PADDLE_ENFORCE_EQ( + param_num, + moment2_out.size(), + errors::InvalidArgument( + "The size of Input(Moment1) must be equal to Input(Param), but got " + "the size of Input(Moment1) is %d, the size of Input(Param) is %d.", + moment2.size(), + param_num)); + PADDLE_ENFORCE_EQ(param_num, + beta1_pow_out.size(), + errors::InvalidArgument( + "The size of Output(Beta1PowOut) must be equal to " + "Input(Param), but got the size of Output(Beta1PowOut) " + "is %d, the size of Input(Param) is %d.", + beta1_pow_out.size(), + param_num)); + PADDLE_ENFORCE_EQ(param_num, + beta2_pow_out.size(), + errors::InvalidArgument( + "The size of Output(Beta2PowOut) must be equal to " + "Input(Param), but got the size of Output(Beta2PowOut) " + "is %d, the size of Input(Param) is %d.", + beta2_pow_out.size(), + param_num)); + PADDLE_ENFORCE_EQ( + param_num, + grad.size(), + errors::InvalidArgument( + "The size of Input(Grad) must be equal to Input(Param), but got " + "the size of Input(Grad) is %d, the size of Input(Param) is %d.", + grad.size(), + param_num)); + PADDLE_ENFORCE_EQ( + param_num, + moment1.size(), + errors::InvalidArgument( + "The size of Input(Moment1) must be equal to Input(Param), but got " + "the size of Input(Moment1) is %d, the size of Input(Param) is %d.", + moment1.size(), + param_num)); + PADDLE_ENFORCE_EQ( + param_num, + moment2.size(), + errors::InvalidArgument( + "The size of Input(Moment1) must be equal to Input(Param), but got " + "the size of Input(Moment1) is %d, the size of Input(Param) is %d.", + moment2.size(), + param_num)); + + std::vector param_list(param_num); + std::vector grad_list(param_num); + std::vector moment1_list(param_num); + std::vector moment2_list(param_num); + std::vector shape_list(param_num); + + for (int j = 0; j < param_num; j++) { + param_list[j] = const_cast(param[j]->data()); + grad_list[j] = const_cast(grad[j]->data()); + moment1_list[j] = const_cast(moment1[j]->data()); + moment2_list[j] = const_cast(moment2[j]->data()); + shape_list[j] = param[j]->numel(); + + PADDLE_ENFORCE_EQ( + param[j], + param_out[j], + errors::InvalidArgument("The size of Input(Param) and Output(ParamOut) " + "must be the same Tensors.")); + PADDLE_ENFORCE_EQ( + moment1[j], + moment1_out[j], + errors::InvalidArgument("The size of Input(Param) and Output(ParamOut) " + "must be the same Tensors.")); + PADDLE_ENFORCE_EQ( + moment2[j], + moment2_out[j], + errors::InvalidArgument("The size of Input(Param) and Output(ParamOut) " + "must be the same Tensors.")); + + dev_ctx.template Alloc(param_out[j]); + dev_ctx.template Alloc(moment1_out[j]); + dev_ctx.template Alloc(moment2_out[j]); + } + + int r = xpu::multi_tensor_adam(dev_ctx.x_context(), + grad_list, + param_list, + moment1_list, + moment2_list, + shape_list, + lr_, + beta1_, + beta2_, + epsilon_, + step_, + mode_, + bias_correction_, + weight_decay_, + beta1_pow_data, + beta2_pow_data); + PADDLE_ENFORCE_XDNN_SUCCESS(r, "merged_adam"); + + // update param, moment1, moment2 + for (int i = 0; i < param_num; i++) { + phi::Copy(dev_ctx, *param[i], dev_ctx.GetPlace(), false, param_out[i]); + phi::Copy(dev_ctx, *moment1[i], dev_ctx.GetPlace(), false, moment1_out[i]); + phi::Copy(dev_ctx, *moment2[i], dev_ctx.GetPlace(), false, moment2_out[i]); + } + + if (!use_global_beta_pow) { + for (int i = 0; i < param_num; i++) { + if (beta1_pow[i]->place() == CPUPlace() && + beta2_pow[i]->place() == CPUPlace()) { + funcs::SetBetaData( + *beta1_pow[i], beta1_pow_out[i], beta1_, dev_ctx); + + funcs::SetBetaData( + *beta2_pow[i], beta2_pow_out[i], beta2_, dev_ctx); + } else { + float* beta1_pow_out_ptr = nullptr; + const float* beta1_pow_data = beta1_pow[i]->data(); + beta1_pow_out_ptr = dev_ctx.template Alloc(beta1_pow_out[i]); + r = xpu::scale(dev_ctx.x_context(), + beta1_pow_data, + beta1_pow_out_ptr, + beta1_pow[i]->numel(), + false, + beta1_, + 0.0f); + PADDLE_ENFORCE_XDNN_SUCCESS(r, "merged_adam"); + + float* beta2_pow_out_ptr = nullptr; + const float* beta2_pow_data = beta2_pow[i]->data(); + beta2_pow_out_ptr = dev_ctx.template Alloc(beta2_pow_out[i]); + r = xpu::scale(dev_ctx.x_context(), + beta2_pow_data, + beta2_pow_out_ptr, + beta2_pow[i]->numel(), + false, + beta2_, + 0.0f); + PADDLE_ENFORCE_XDNN_SUCCESS(r, "merged_adam"); + } + } + } +} } // namespace phi PD_REGISTER_KERNEL( @@ -252,3 +475,11 @@ PD_REGISTER_KERNEL( kernel->OutputAt(3).SetBackend(phi::Backend::UNDEFINED); kernel->OutputAt(4).SetBackend(phi::Backend::UNDEFINED); } + +PD_REGISTER_KERNEL(merged_adam, XPU, ALL_LAYOUT, phi::MergedAdamKernel, float) { + // Skip beta1_pow, beta2_pow data transform + kernel->InputAt(5).SetBackend(phi::Backend::ALL_BACKEND); + kernel->InputAt(6).SetBackend(phi::Backend::ALL_BACKEND); + kernel->OutputAt(3).SetBackend(phi::Backend::UNDEFINED); + kernel->OutputAt(4).SetBackend(phi::Backend::UNDEFINED); +} diff --git a/test/xpu/test_adam_op_xpu.py b/test/xpu/test_adam_op_xpu.py index 823cf1543e0a50..54f8d36a187a4a 100644 --- a/test/xpu/test_adam_op_xpu.py +++ b/test/xpu/test_adam_op_xpu.py @@ -271,7 +271,9 @@ def adam_step(inputs, attributes): moment1_out = beta1 * moment1 + (1 - beta1) * grad moment2_out = beta2 * moment2 + (1 - beta2) * np.square(grad) lr_t = lr * np.sqrt(1 - beta2_pow) / (1 - beta1_pow) - param_out = param - lr_t * (moment1_out / (np.sqrt(moment2_out) + epsilon)) + param_out = param - lr_t * ( + moment1_out / (np.sqrt(moment2_out) + epsilon * np.sqrt(1 - beta2_pow)) + ) return param_out, moment1_out, moment2_out diff --git a/test/xpu/test_merged_adam_op_xpu.py b/test/xpu/test_merged_adam_op_xpu.py new file mode 100644 index 00000000000000..2fb07f27f923ef --- /dev/null +++ b/test/xpu/test_merged_adam_op_xpu.py @@ -0,0 +1,279 @@ +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import numpy as np +from get_test_cover_info import ( + XPUOpTestWrapper, + create_test_class, + get_xpu_op_support_types, +) + +import paddle + +# from op import Operator +# from op_test_xpu import XPUOpTest +from paddle import _C_ops, _legacy_C_ops +from paddle.base.framework import in_dygraph_mode + + +def run_adam_op( + params, + grads, + lrs, + moment1s, + moment2s, + beta1_pows, + beta2_pows, + master_params, + epsilon, + beta1, + beta2, + place, + multi_precision=False, + use_merged=False, +): + assert len(params) == len(grads) + assert len(params) == len(lrs) + assert len(params) == len(moment1s) + assert len(params) == len(moment2s) + assert len(params) == len(beta1_pows) + assert len(params) == len(beta1_pows) + assert len(params) == len(master_params) + paddle.disable_static() + paddle.set_device(place) + + param_vars = [paddle.base.dygraph.to_variable(p) for p in params] + grad_vars = [paddle.base.dygraph.to_variable(g) for g in grads] + lr_vars = [paddle.base.dygraph.to_variable(l) for l in lrs] + moment1_vars = [paddle.base.dygraph.to_variable(m) for m in moment1s] + moment2_vars = [paddle.base.dygraph.to_variable(m) for m in moment2s] + beta1_pow_vars = [paddle.base.dygraph.to_variable(b) for b in beta1_pows] + beta2_pow_vars = [paddle.base.dygraph.to_variable(b) for b in beta2_pows] + master_param_vars = [ + paddle.base.dygraph.to_variable(m_p) for m_p in master_params + ] + + if not use_merged: + for i in range(len(param_vars)): + _, _, _, _, _, _ = _legacy_C_ops.adam( + param_vars[i], + grad_vars[i], + lr_vars[i], + moment1_vars[i], + moment2_vars[i], + beta1_pow_vars[i], + beta2_pow_vars[i], + master_param_vars[i], + param_vars[i], + moment1_vars[i], + moment2_vars[i], + beta1_pow_vars[i], + beta2_pow_vars[i], + master_param_vars[i], + 'epsilon', + epsilon, + 'beta1', + beta1, + 'beta2', + beta2, + 'multi_precision', + False, + ) + else: + if in_dygraph_mode(): + _, _, _, _, _, _ = _C_ops.merged_adam_( + param_vars, + grad_vars, + lr_vars, + moment1_vars, + moment2_vars, + beta1_pow_vars, + beta2_pow_vars, + master_param_vars, + beta1, + beta2, + epsilon, + False, + False, + ) + else: + _, _, _, _, _, _ = _legacy_C_ops.merged_adam( + param_vars, + grad_vars, + lr_vars, + moment1_vars, + moment2_vars, + beta1_pow_vars, + beta2_pow_vars, + master_param_vars, + param_vars, + moment1_vars, + moment2_vars, + beta1_pow_vars, + beta2_pow_vars, + master_param_vars, + 'epsilon', + epsilon, + 'beta1', + beta1, + 'beta2', + beta2, + 'multi_precision', + multi_precision, + ) + + outputs = { + 'ParamOut': param_vars, + 'Moment1Out': moment1_vars, + 'Moment2Out': moment2_vars, + 'Beta1PowOut': beta1_pow_vars, + 'Beta2PowOut': beta2_pow_vars, + 'MasterParamOut': master_param_vars, + } + + return outputs + + +class XPUTestMergedAdamWrapper(XPUOpTestWrapper): + def __init__(self): + self.op_name = 'merged_adam' + self.use_dynamic_create_class = False + + class XPUTestMergedAdamBase(unittest.TestCase): + def setUp(self): + self.shapes = [[3, 4], [2, 7], [5, 6], [7, 8]] + self.seed = 10 + + def gen_rand_data(self, shapes, dtype): + return [np.random.random(s).astype(dtype) for s in shapes] + + def prepare_data(self, shapes, seed): + np.random.seed(seed) + mp_dtype = np.float32 + dtype = np.float32 + params = self.gen_rand_data(shapes, dtype) + grads = self.gen_rand_data(shapes, dtype) + learning_rate = self.gen_rand_data([[1]], mp_dtype) + lrs = [learning_rate.copy() for _ in shapes] + moment1s = self.gen_rand_data(shapes, mp_dtype) + moment2s = self.gen_rand_data(shapes, mp_dtype) + beta1_pow = self.gen_rand_data([[1]], mp_dtype) + beta2_pow = self.gen_rand_data([[1]], mp_dtype) + beta1_pows = [beta1_pow.copy() for _ in shapes] + beta2_pows = [beta2_pow.copy() for _ in shapes] + master_params = [p.astype(mp_dtype) for p in params] + return ( + params, + grads, + lrs, + moment1s, + moment2s, + beta1_pows, + beta2_pows, + master_params, + ) + + def check_with_place(self): + ( + params, + grads, + lrs, + moment1s, + moment2s, + beta1_pows, + beta2_pows, + master_params, + ) = self.prepare_data(self.shapes, self.seed) + + def run_op(use_merged, place): + return run_adam_op( + params=params, + grads=grads, + lrs=lrs, + moment1s=moment1s, + moment2s=moment2s, + beta1_pows=beta1_pows, + beta2_pows=beta2_pows, + master_params=master_params, + epsilon=0.9, + beta1=0.9, + beta2=0.99, + place=place, + multi_precision=False, + use_merged=use_merged, + ) + + outs1 = run_op(True, "xpu") + outs2 = run_op(True, "cpu") + outs3 = run_op(False, "xpu") + outs4 = run_op(False, "cpu") + + self.assertEqual(len(outs1), len(outs2)) + self.assertEqual(len(outs1), len(outs3)) + self.assertEqual(len(outs1), len(outs4)) + + for key in outs1.keys(): + value1 = outs1[key] + value2 = outs2[key] + value3 = outs3[key] + value4 = outs4[key] + for i in range(len(value1)): + np.testing.assert_allclose( + value1[i], value2[i], rtol=1e-05, atol=1e-07 + ) + np.testing.assert_allclose( + value1[i], value3[i], rtol=1e-05, atol=1e-07 + ) + np.testing.assert_allclose( + value1[i], value4[i], rtol=1e-05, atol=1e-07 + ) + + class TestMergedAdamOp(XPUTestMergedAdamBase): + def setUp(self): + super().setUp() + self.set_case() + + def set_case(self): + self.shapes = [[3, 4], [2, 7], [5, 6], [7, 8]] + self.seed = 10 + + def testalltype(self): + self.check_with_place() + + class TestMergedAdam1(TestMergedAdamOp): + def set_case(self): + self.shapes = [[3, 4]] + + class TestMergedAdam2(TestMergedAdamOp): + def set_case(self): + self.shapes = [[3, 4], [2, 7]] + + class TestMergedAdam3(TestMergedAdamOp): + def set_case(self): + self.shapes = [[3, 4], [2, 4], [3, 4]] + + class TestMergedAdam4(TestMergedAdamOp): + def set_case(self): + self.shapes = [[3, 4], [2, 7], [5, 6], [9, 9]] + + +support_types = get_xpu_op_support_types('merged_adam') +for stype in support_types: + create_test_class(globals(), XPUTestMergedAdamWrapper, stype) + +if __name__ == "__main__": + paddle.enable_static() + unittest.main() From d963050dd82cfb55a659c620aad3bef0e84d634c Mon Sep 17 00:00:00 2001 From: LoneRanger <836253168@qq.com> Date: Mon, 20 Nov 2023 14:19:59 +0800 Subject: [PATCH 15/46] =?UTF-8?q?=E3=80=90PIR=20API=20adaptor=20No.45-47?= =?UTF-8?q?=E3=80=91Migrate=20some=20ops=20into=20pir=20(#58682)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- python/paddle/nn/functional/loss.py | 2 +- python/paddle/tensor/linalg.py | 2 +- python/paddle/tensor/manipulation.py | 12 +++-- test/legacy_test/test_crop_tensor_op.py | 8 +-- test/legacy_test/test_cross_op.py | 38 ++++++++++---- .../test_softmax_with_cross_entropy_op.py | 51 ++++++++++++------- 6 files changed, 74 insertions(+), 39 deletions(-) diff --git a/python/paddle/nn/functional/loss.py b/python/paddle/nn/functional/loss.py index fa27a4fedbd3cc..ab331cca7a95ab 100644 --- a/python/paddle/nn/functional/loss.py +++ b/python/paddle/nn/functional/loss.py @@ -271,7 +271,7 @@ def base_softmax_with_cross_entropy( ) if input_dims - 1 == label_dims: label = paddle.unsqueeze(label, axis=axis) - if in_dynamic_mode(): + if in_dynamic_or_pir_mode(): softmax, loss = _C_ops.cross_entropy_with_softmax( logits, label, diff --git a/python/paddle/tensor/linalg.py b/python/paddle/tensor/linalg.py index 13cff344180f9c..d4aba965daf51d 100644 --- a/python/paddle/tensor/linalg.py +++ b/python/paddle/tensor/linalg.py @@ -1467,7 +1467,7 @@ def cross(x, y, axis=9, name=None): [0., 0., 0.], [0., 0., 0.]]) """ - if in_dynamic_mode(): + if in_dynamic_or_pir_mode(): axis = K_DEFAULT_DIM if axis is None else axis return _C_ops.cross(x, y, axis) else: diff --git a/python/paddle/tensor/manipulation.py b/python/paddle/tensor/manipulation.py index 5edcb8133f03f7..685b10276c476f 100644 --- a/python/paddle/tensor/manipulation.py +++ b/python/paddle/tensor/manipulation.py @@ -781,10 +781,16 @@ def crop(x, shape=None, offsets=None, name=None): x, 'x', ['float32', 'float64', 'int32', 'int64'], 'crop_tensor' ) check_type( - shape, 'shape', (list, tuple, Variable, type(None)), 'crop_tensor' + shape, + 'shape', + (list, tuple, Variable, type(None), paddle.pir.OpResult), + 'crop_tensor', ) check_type( - offsets, 'offsets', (list, tuple, Variable, type(None)), 'crop_tensor' + offsets, + 'offsets', + (list, tuple, Variable, type(None), paddle.pir.OpResult), + 'crop_tensor', ) if offsets is None: @@ -793,7 +799,7 @@ def crop(x, shape=None, offsets=None, name=None): if shape is None: shape = x.shape - if in_dynamic_mode(): + if in_dynamic_or_pir_mode(): return _C_ops.crop(x, shape, offsets) out = helper.create_variable_for_type_inference(x.dtype) diff --git a/test/legacy_test/test_crop_tensor_op.py b/test/legacy_test/test_crop_tensor_op.py index 8d9743a55171c4..ab8a6466d97a52 100644 --- a/test/legacy_test/test_crop_tensor_op.py +++ b/test/legacy_test/test_crop_tensor_op.py @@ -81,10 +81,10 @@ def initTestCase(self): self.offsets = [1, 2] def test_check_output(self): - self.check_output() + self.check_output(check_pir=True) def test_check_grad_normal(self): - self.check_grad(['X'], 'Out') + self.check_grad(['X'], 'Out', check_pir=True) class TestCase1(TestCropTensorOp): @@ -182,10 +182,10 @@ def initTestCase(self): self.shape_attr = [0, 0] def test_check_output(self): - self.check_output() + self.check_output(check_pir=True) def test_check_grad_normal(self): - self.check_grad(["X"], "Out") + self.check_grad(["X"], "Out", check_pir=True) class TestCropTensorOpTensorAttrCase1(TestCropTensorOpTensorAttr): diff --git a/test/legacy_test/test_cross_op.py b/test/legacy_test/test_cross_op.py index cd13ea10f45106..6aeab30d6c42f7 100644 --- a/test/legacy_test/test_cross_op.py +++ b/test/legacy_test/test_cross_op.py @@ -19,7 +19,8 @@ import paddle from paddle import base -from paddle.base import Program, core, program_guard +from paddle.base import core +from paddle.pir_utils import test_with_pir_api class TestCrossOp(OpTest): @@ -47,10 +48,10 @@ def init_output(self): self.outputs = {'Out': np.array(z_list).reshape(self.shape)} def test_check_output(self): - self.check_output() + self.check_output(check_pir=True) def test_check_grad_normal(self): - self.check_grad(['X', 'Y'], 'Out') + self.check_grad(['X', 'Y'], 'Out', check_pir=True) class TestCrossOpCase1(TestCrossOp): @@ -116,13 +117,15 @@ def test_check_output(self): if core.is_compiled_with_cuda(): place = core.CUDAPlace(0) if core.is_bfloat16_supported(place): - self.check_output_with_place(place) + self.check_output_with_place(place, check_pir=True) def test_check_grad_normal(self): if core.is_compiled_with_cuda(): place = core.CUDAPlace(0) if core.is_bfloat16_supported(place): - self.check_grad_with_place(place, ['X', 'Y'], 'Out') + self.check_grad_with_place( + place, ['X', 'Y'], 'Out', check_pir=True + ) class TestCrossAPI(unittest.TestCase): @@ -134,18 +137,22 @@ def input_data(self): [[1.0, 1.0, 1.0], [1.0, 1.0, 1.0], [1.0, 1.0, 1.0]] ).astype('float32') + @test_with_pir_api def test_cross_api(self): self.input_data() + main = paddle.static.Program() + startup = paddle.static.Program() # case 1: - with program_guard(Program(), Program()): + with paddle.static.program_guard(main, startup): x = paddle.static.data(name='x', shape=[-1, 3], dtype="float32") y = paddle.static.data(name='y', shape=[-1, 3], dtype="float32") z = paddle.cross(x, y, axis=1) exe = base.Executor(base.CPUPlace()) (res,) = exe.run( + main, feed={'x': self.data_x, 'y': self.data_y}, - fetch_list=[z.name], + fetch_list=[z], return_numpy=False, ) expect_out = np.array( @@ -153,15 +160,18 @@ def test_cross_api(self): ) np.testing.assert_allclose(expect_out, np.array(res), rtol=1e-05) + main = paddle.static.Program() + startup = paddle.static.Program() # case 2: - with program_guard(Program(), Program()): + with paddle.static.program_guard(main, startup): x = paddle.static.data(name='x', shape=[-1, 3], dtype="float32") y = paddle.static.data(name='y', shape=[-1, 3], dtype="float32") z = paddle.cross(x, y) exe = base.Executor(base.CPUPlace()) (res,) = exe.run( + main, feed={'x': self.data_x, 'y': self.data_y}, - fetch_list=[z.name], + fetch_list=[z], return_numpy=False, ) expect_out = np.array( @@ -169,8 +179,14 @@ def test_cross_api(self): ) np.testing.assert_allclose(expect_out, np.array(res), rtol=1e-05) - # case 3: - with program_guard(Program(), Program()): + def test_cross_api1(self): + self.input_data() + + main = paddle.static.Program() + startup = paddle.static.Program() + + # case 1: + with paddle.static.program_guard(main, startup): x = paddle.static.data(name="x", shape=[-1, 3], dtype="float32") y = paddle.static.data(name='y', shape=[-1, 3], dtype='float32') diff --git a/test/legacy_test/test_softmax_with_cross_entropy_op.py b/test/legacy_test/test_softmax_with_cross_entropy_op.py index 0515c7c78e9d70..e2d512707e57de 100644 --- a/test/legacy_test/test_softmax_with_cross_entropy_op.py +++ b/test/legacy_test/test_softmax_with_cross_entropy_op.py @@ -153,27 +153,30 @@ def setUp(self): def test_check_output(self): if self.python_api is not None: - self.check_output() - self.check_output() + self.check_output(check_pir=True) + self.check_output(check_pir=True) def test_check_grad(self): if core.is_compiled_with_rocm(): if self.python_api is not None: self.check_grad( - ["Logits"], - "Loss", - max_relative_error=5e-1, + ["Logits"], "Loss", max_relative_error=5e-1, check_pir=False ) # HIP will have accuracy fail when using float32 in CPU place - self.check_grad(["Logits"], "Loss", max_relative_error=5e-1) + self.check_grad( + ["Logits"], "Loss", max_relative_error=5e-1, check_pir=False + ) else: if self.python_api is not None: self.check_grad( ["Logits"], "Loss", numeric_grad_delta=0.001, + check_pir=False, ) - self.check_grad(["Logits"], "Loss", numeric_grad_delta=0.001) + self.check_grad( + ["Logits"], "Loss", numeric_grad_delta=0.001, check_pir=False + ) class TestSoftmaxWithCrossEntropyOpInt32(TestSoftmaxWithCrossEntropyOp): @@ -509,13 +512,15 @@ def setUp(self): def test_check_output(self): if self.python_api is not None: - self.check_output() - self.check_output() + self.check_output(check_pir=True) + self.check_output(check_pir=True) def test_check_grad(self): if self.python_api is not None: - self.check_grad(["Logits"], "Loss") - self.check_grad(["Logits"], "Loss", max_relative_error=0.1) + self.check_grad(["Logits"], "Loss", check_pir=False) + self.check_grad( + ["Logits"], "Loss", max_relative_error=0.1, check_pir=False + ) class TestSoftmaxWithCrossEntropyOpNoCudnnFp16( @@ -534,8 +539,12 @@ def initParams(self): def test_check_grad(self): if self.python_api is not None: - self.check_grad(["Logits"], "Loss", max_relative_error=0.1) - self.check_grad(["Logits"], "Loss", max_relative_error=0.1) + self.check_grad( + ["Logits"], "Loss", max_relative_error=0.1, check_pir=False + ) + self.check_grad( + ["Logits"], "Loss", max_relative_error=0.1, check_pir=False + ) class TestSoftmaxWithCrossEntropyOp2(TestSoftmaxWithCrossEntropyOp): @@ -557,19 +566,23 @@ def initParams(self): def test_check_output(self): if self.python_api is not None: - self.check_output() - self.check_output() + self.check_output(check_pir=True) + self.check_output(check_pir=True) def test_check_grad(self): if core.is_compiled_with_rocm(): # HIP will have accuracy fail when using float32 in CPU place if self.python_api is not None: - self.check_grad(["Logits"], "Loss", max_relative_error=0.1) - self.check_grad(["Logits"], "Loss", max_relative_error=0.1) + self.check_grad( + ["Logits"], "Loss", max_relative_error=0.1, check_pir=False + ) + self.check_grad( + ["Logits"], "Loss", max_relative_error=0.1, check_pir=False + ) else: if self.python_api is not None: - self.check_grad(["Logits"], "Loss") - self.check_grad(["Logits"], "Loss") + self.check_grad(["Logits"], "Loss", check_pir=False) + self.check_grad(["Logits"], "Loss", check_pir=False) class TestSoftmaxWithCrossEntropyOp3(TestSoftmaxWithCrossEntropyOp): From 3525e9b1bbbd37ca6553e4ed9945ec4806379568 Mon Sep 17 00:00:00 2001 From: winter-wang <78149749+winter-wang@users.noreply.github.com> Date: Mon, 20 Nov 2023 14:38:54 +0800 Subject: [PATCH 16/46] [PIR]polish iterator of block®ion. (#59118) --- .../hlir/dialect/operator/ir/manual_op.cc | 8 +- .../transforms/cinn_group_lowering_pass.cc | 26 +-- paddle/cinn/hlir/framework/pir_compiler.cc | 11 +- .../eager/to_static/run_program_op_node.h | 26 +-- .../instruction/instruction_util.cc | 32 +-- .../interpreter/interpreter_util.cc | 22 +- .../pir_adaptor/pir_adaptor_util.cc | 24 +-- .../framework/new_executor/pir_interpreter.cc | 41 ++-- .../new_executor/standalone_executor.cc | 12 +- .../translator/program_translator.cc | 2 +- .../dialect/operator/ir/control_flow_op.cc | 12 +- paddle/fluid/pir/drr/drr_rewrite_pattern.cc | 6 +- .../fluid/pir/transforms/build_cinn_pass.cc | 22 +- paddle/fluid/pir/transforms/inplace_pass.cc | 107 +++++----- .../params_sync_among_devices_pass.cc | 10 +- .../pir/transforms/pd_op_to_kernel_pass.cc | 34 ++-- paddle/fluid/pybind/control_flow_api.cc | 92 +++++++-- paddle/fluid/pybind/control_flow_api.h | 6 - paddle/fluid/pybind/pir.cc | 41 ++-- paddle/pir/core/block.cc | 14 +- paddle/pir/core/block.h | 21 +- paddle/pir/core/ir_printer.cc | 8 +- paddle/pir/core/ir_printer.h | 2 +- paddle/pir/core/iterator.h | 190 ++++++++++++++++++ paddle/pir/core/op_base.h | 3 + paddle/pir/core/operation.cc | 17 +- paddle/pir/core/operation.h | 20 +- paddle/pir/core/region.cc | 10 +- paddle/pir/core/region.h | 27 ++- paddle/pir/core/use_iterator.h | 55 ----- paddle/pir/core/value.h | 2 +- paddle/pir/core/verify.cc | 9 +- paddle/pir/dialect/shape/ir/shape_op.cc | 4 +- .../shape/transforms/shape_optimization.cc | 12 +- .../shape/utils/shape_optimization_utils.cc | 44 ++-- paddle/pir/dialect/shape/utils/shape_utils.cc | 4 +- paddle/pir/pass/pass.cc | 8 +- .../pattern_rewrite/pattern_rewrite_driver.cc | 12 +- .../cinn/add_broadcast_to_elementwise_test.cc | 28 +-- test/cpp/pir/cinn/build_cinn_pass_test.cc | 20 +- test/cpp/pir/cinn/dialect_convert_test.cc | 16 +- test/cpp/pir/cinn/group_op_test.cc | 12 +- test/cpp/pir/cinn/ir_op_fusion_test.cc | 97 +++++---- test/cpp/pir/cinn/jit_instruction_test.cc | 26 +-- test/cpp/pir/core/program_translator_test.cc | 88 ++++---- test/cpp/pir/pass/pass_manager_test.cc | 6 +- .../drr_fuse_linear_param_grad_add_test.cc | 4 +- test/cpp/prim/test_vjp.cc | 6 +- test/ir/pir/test_if_api.py | 9 +- 49 files changed, 764 insertions(+), 544 deletions(-) create mode 100644 paddle/pir/core/iterator.h delete mode 100644 paddle/pir/core/use_iterator.h diff --git a/paddle/cinn/hlir/dialect/operator/ir/manual_op.cc b/paddle/cinn/hlir/dialect/operator/ir/manual_op.cc index 0ba418dbec811c..3a7341602a7641 100644 --- a/paddle/cinn/hlir/dialect/operator/ir/manual_op.cc +++ b/paddle/cinn/hlir/dialect/operator/ir/manual_op.cc @@ -54,9 +54,11 @@ pir::Block *GroupOp::block() { } std::vector GroupOp::ops() { - auto *inner_block = this->block(); - return std::vector(inner_block->begin(), - inner_block->end()); + std::vector rt_ops; + for (auto &op : *block()) { + rt_ops.push_back(&op); + } + return rt_ops; } void GroupOp::VerifySig() {} diff --git a/paddle/cinn/hlir/dialect/operator/transforms/cinn_group_lowering_pass.cc b/paddle/cinn/hlir/dialect/operator/transforms/cinn_group_lowering_pass.cc index 4fab401b2e1e3a..616f2ef222eef8 100644 --- a/paddle/cinn/hlir/dialect/operator/transforms/cinn_group_lowering_pass.cc +++ b/paddle/cinn/hlir/dialect/operator/transforms/cinn_group_lowering_pass.cc @@ -141,9 +141,9 @@ std::unique_ptr CINNGroupLoweringPass(::pir::Program* program) { for (auto it = program->block()->begin(); it != program->block()->end(); ++it) { - if ((*it)->isa()) { + if (it->isa()) { // GetOpList and Call cinn CodeGen - auto group_op = (*it)->dyn_cast(); + auto group_op = it->dyn_cast(); // op fusion auto op_fusion = cinn::dialect::ir::OpFusionPassInternal( @@ -204,26 +204,26 @@ std::unique_ptr CINNGroupLoweringPass(::pir::Program* program) { } else { std::vector vec_ins; - for (size_t i = 0; i < (*it)->num_operands(); ++i) { - if ((*it)->operand_source(i)) { - vec_ins.push_back(value_map.at((*it)->operand_source(i))); + for (size_t i = 0; i < it->num_operands(); ++i) { + if (it->operand_source(i)) { + vec_ins.push_back(value_map.at(it->operand_source(i))); } else { - vec_ins.push_back((*it)->operand_source(i)); + vec_ins.push_back(it->operand_source(i)); } } std::vector vec_types; - for (size_t i = 0; i < (*it)->num_results(); ++i) { - vec_types.push_back((*it)->result(i).type()); + for (size_t i = 0; i < it->num_results(); ++i) { + vec_types.push_back(it->result(i).type()); } - ::pir::OpInfo info1 = ctx->GetRegisteredOpInfo((*it)->name()); - ::pir::Operation* op = ::pir::Operation::Create( - vec_ins, (*it)->attributes(), vec_types, info1); + ::pir::OpInfo info1 = ctx->GetRegisteredOpInfo(it->name()); + ::pir::Operation* op = + ::pir::Operation::Create(vec_ins, it->attributes(), vec_types, info1); ir_program->block()->push_back(op); - for (size_t i = 0; i < (*it)->num_results(); ++i) { - value_map[(*it)->result(i)] = op->result(i); + for (size_t i = 0; i < it->num_results(); ++i) { + value_map[it->result(i)] = op->result(i); } } } diff --git a/paddle/cinn/hlir/framework/pir_compiler.cc b/paddle/cinn/hlir/framework/pir_compiler.cc index bea57a651bed25..1ad5e921d314a9 100644 --- a/paddle/cinn/hlir/framework/pir_compiler.cc +++ b/paddle/cinn/hlir/framework/pir_compiler.cc @@ -29,9 +29,8 @@ std::unique_ptr PirCompiler::Build() { m_builder_.Clear(); // NOTE(Aurelius84): Currently only support each op for one group std::vector groups; - for (auto it = program_.block()->begin(); it != program_.block()->end(); - ++it) { - std::vector<::pir::Operation*> ops = {*it}; + for (auto& op : *program_.block()) { + std::vector<::pir::Operation*> ops = {&op}; groups.push_back(std::make_shared(ops)); } VLOG(4) << "Groups size: " << groups.size(); @@ -185,12 +184,12 @@ std::shared_ptr BuildScope(const Target& target, tensor->set_type(pir::CompatibleInfo::ConvertIRType(type_info.dtype())); }; - for (auto it = program.block()->begin(); it != program.block()->end(); ++it) { - for (auto& oprand : (*it)->operands()) { + for (auto& op : *program.block()) { + for (auto oprand : op.operands()) { create_var(oprand.source()); } - for (auto& result : (*it)->results()) { + for (auto result : op.results()) { create_var(result); } } diff --git a/paddle/fluid/eager/to_static/run_program_op_node.h b/paddle/fluid/eager/to_static/run_program_op_node.h index bb398391ea37dd..9b90c664b11776 100644 --- a/paddle/fluid/eager/to_static/run_program_op_node.h +++ b/paddle/fluid/eager/to_static/run_program_op_node.h @@ -181,30 +181,30 @@ static auto GetNameFromValue(const ::pir::Block *block, bool is_input) { // we use name here, later value is used directly. std::unordered_map<::pir::Value, std::string> value2name; - for (auto *op : *block) { + for (auto &op : *block) { std::string name; - if (is_input && op->name() == "pd_op.data") { + if (is_input && op.name() == "pd_op.data") { name = - op->attributes().at("name").dyn_cast().AsString(); - value2name[op->results()[0].Value::impl()] = name; - } else if (!is_input && op->name() == "builtin.set_parameter") { - name = op->attributes() + op.attributes().at("name").dyn_cast().AsString(); + value2name[op.results()[0].Value::impl()] = name; + } else if (!is_input && op.name() == "builtin.set_parameter") { + name = op.attributes() .at("parameter_name") .dyn_cast() .AsString(); - value2name[op->operand(0).source()] = name; - } else if (!is_input && op->name() == "builtin.shadow_output") { - name = op->attributes() + value2name[op.operand(0).source()] = name; + } else if (!is_input && op.name() == "builtin.shadow_output") { + name = op.attributes() .at("output_name") .dyn_cast() .AsString(); - value2name[op->operand(0).source()] = name; - } else if (is_input && op->name() == "builtin.get_parameter") { - name = op->attributes() + value2name[op.operand(0).source()] = name; + } else if (is_input && op.name() == "builtin.get_parameter") { + name = op.attributes() .at("parameter_name") .dyn_cast() .AsString(); - value2name[op->result(0).Value::impl()] = name; + value2name[op.result(0).Value::impl()] = name; } } std::vector names; diff --git a/paddle/fluid/framework/new_executor/instruction/instruction_util.cc b/paddle/fluid/framework/new_executor/instruction/instruction_util.cc index f8174fb7d179ad..ebf46ab6f7cd3c 100644 --- a/paddle/fluid/framework/new_executor/instruction/instruction_util.cc +++ b/paddle/fluid/framework/new_executor/instruction/instruction_util.cc @@ -228,19 +228,19 @@ std::unordered_set GetBlockInnerOutputs(pir::Block* block) { for (size_t arg_id = 0; arg_id < block->args_size(); ++arg_id) { inner_outputs.insert(block->argument(arg_id)); } - for (auto op : (*block)) { - VLOG(8) << "GetBlockInnerOutputs of " << op->name(); - if (op->num_regions()) { - for (size_t i = 0; i < op->num_regions(); ++i) { - for (auto sub_block : op->region(i)) { + for (auto& op : (*block)) { + VLOG(8) << "GetBlockInnerOutputs of " << op.name(); + if (op.num_regions()) { + for (size_t i = 0; i < op.num_regions(); ++i) { + for (auto& sub_block : op.region(i)) { std::unordered_set sub_set = - GetBlockInnerOutputs(sub_block); + GetBlockInnerOutputs(&sub_block); inner_outputs.insert(sub_set.begin(), sub_set.end()); } } } - for (size_t i = 0; i < op->num_results(); ++i) { - inner_outputs.insert(op->result(i)); + for (size_t i = 0; i < op.num_results(); ++i) { + inner_outputs.insert(op.result(i)); } } return inner_outputs; @@ -248,19 +248,19 @@ std::unordered_set GetBlockInnerOutputs(pir::Block* block) { std::unordered_set GetBlockInnerInputs(pir::Block* block) { std::unordered_set inner_inputs; - for (auto op : (*block)) { - VLOG(8) << "GetBlockInnerInputs of " << op->name(); - if (op->num_regions()) { - for (size_t i = 0; i < op->num_regions(); ++i) { - for (auto sub_block : op->region(i)) { + for (auto& op : (*block)) { + VLOG(8) << "GetBlockInnerInputs of " << op.name(); + if (op.num_regions()) { + for (size_t i = 0; i < op.num_regions(); ++i) { + for (auto& sub_block : op.region(i)) { std::unordered_set sub_set = - GetBlockInnerInputs(sub_block); + GetBlockInnerInputs(&sub_block); inner_inputs.insert(sub_set.begin(), sub_set.end()); } } } - for (size_t i = 0; i < op->num_operands(); ++i) { - inner_inputs.insert(op->operand_source(i)); + for (size_t i = 0; i < op.num_operands(); ++i) { + inner_inputs.insert(op.operand_source(i)); } } return inner_inputs; diff --git a/paddle/fluid/framework/new_executor/interpreter/interpreter_util.cc b/paddle/fluid/framework/new_executor/interpreter/interpreter_util.cc index ee8c8cd2ec79da..52d921db03b15a 100644 --- a/paddle/fluid/framework/new_executor/interpreter/interpreter_util.cc +++ b/paddle/fluid/framework/new_executor/interpreter/interpreter_util.cc @@ -1238,15 +1238,15 @@ void PrintValuesAndVariables( const std::unordered_map& value_2_var_name, const std::unordered_map& variable_2_var_name) { - for (const auto& op : block) { + for (auto& op : block) { std::stringstream ss; VLOG(6) << "-----------------------------"; - op->Print(ss); + op.Print(ss); VLOG(6) << ss.str(); - std::string op_name = op->name(); - if (op->attributes().count("op_name")) { - op_name = op->attributes() + std::string op_name = op.name(); + if (op.attributes().count("op_name")) { + op_name = op.attributes() .at("op_name") .dyn_cast() .AsString(); @@ -1258,9 +1258,9 @@ void PrintValuesAndVariables( // 1. output string std::string ret_value_str = "Value : ("; std::string ret_variable_str = "Variable: ("; - if (!op->results().empty()) { - for (size_t i = 0; i < op->num_results(); ++i) { - pir::Value out_value = op->result(i); + if (!op.results().empty()) { + for (size_t i = 0; i < op.num_results(); ++i) { + pir::Value out_value = op.result(i); if (value_2_var_name.count(out_value)) { // get Variable by Value auto& var_name = value_2_var_name.at(out_value); @@ -1307,9 +1307,9 @@ void PrintValuesAndVariables( // 3. input string ret_value_str += "("; ret_variable_str += "("; - if (!op->operands().empty()) { - for (size_t i = 0; i < op->num_operands(); ++i) { - ::pir::Value in_value = op->operand(i).source(); + if (!op.operands().empty()) { + for (size_t i = 0; i < op.num_operands(); ++i) { + ::pir::Value in_value = op.operand(i).source(); if (value_2_var_name.count(in_value)) { // get Variable by Value auto& var_name = value_2_var_name.at(in_value); diff --git a/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.cc b/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.cc index 890b4ef74c9dc6..f516195caad16e 100644 --- a/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.cc +++ b/paddle/fluid/framework/new_executor/pir_adaptor/pir_adaptor_util.cc @@ -561,33 +561,33 @@ void BuildScope(const pir::Block& block, << GenScopeTreeDebugInfo( const_cast(value_exe_info->GetScope()->root())); - for (auto op : block) { - std::string op_name = op->name(); - if (op->attributes().count("op_name")) { - op_name = op->attributes() + for (auto& op : block) { + std::string op_name = op.name(); + if (op.attributes().count("op_name")) { + op_name = op.attributes() .at("op_name") .dyn_cast() .AsString(); } VLOG(4) << "build op:" << op_name; if (SpecialOps.count(op_name)) { - HandleForSpecialOp(op, var_name_prefix, value_exe_info); + HandleForSpecialOp(&op, var_name_prefix, value_exe_info); continue; } - CheckInputVars(op, op_name, value_exe_info); + CheckInputVars(&op, op_name, value_exe_info); - if (op->num_results() < 1) continue; - if (op->attributes().count("is_inplace") != 0 && - op->attributes() + if (op.num_results() < 1) continue; + if (op.attributes().count("is_inplace") != 0 && + op.attributes() .at("is_inplace") .dyn_cast() .data()) { - HandleForInplaceOp(op, var_name_prefix, value_exe_info); + HandleForInplaceOp(&op, var_name_prefix, value_exe_info); continue; } else { - for (size_t i = 0; i < op->num_results(); ++i) { - BuildValue(op->result(i), var_name_prefix, value_exe_info); + for (size_t i = 0; i < op.num_results(); ++i) { + BuildValue(op.result(i), var_name_prefix, value_exe_info); } } } diff --git a/paddle/fluid/framework/new_executor/pir_interpreter.cc b/paddle/fluid/framework/new_executor/pir_interpreter.cc index 28b587d2c7c9eb..5d9eacaa077e0d 100644 --- a/paddle/fluid/framework/new_executor/pir_interpreter.cc +++ b/paddle/fluid/framework/new_executor/pir_interpreter.cc @@ -637,27 +637,32 @@ void PirInterpreter::BuildInstruction() { size_t op_idx = 0; for (auto& op : *ir_block_) { VLOG(6) << "Build Instruction for op: " << op_idx; - if (op->dialect()->name() == "builtin") { - if (interpreter::GetSpecialOpNames().count(op->name())) { - VLOG(6) << "skip process builtin dialect op: " << op->name(); + if (op.dialect()->name() == "builtin") { + if (interpreter::GetSpecialOpNames().count(op.name())) { + VLOG(6) << "skip process builtin dialect op: " << op.name(); continue; } - } else if (op->dialect()->name() == "cf") { - VLOG(6) << "skip process cf dialect op: " << op->name(); + } else if (op.dialect()->name() == "cf") { + VLOG(6) << "skip process cf dialect op: " << op.name(); continue; - } else if (op->dialect()->name() == "pd_op") { - if (op->isa()) { + } else if (op.dialect()->name() == "pd_op") { + if (op.isa()) { vec_instruction_base_.emplace_back(std::make_unique( - op_idx++, place_, op, value_exe_info_.get())); - } else if (op->isa()) { - vec_instruction_base_.emplace_back(std::make_unique( - op_idx++, place_, op, scope_, local_scope_, value_exe_info_.get())); + op_idx++, place_, &op, value_exe_info_.get())); + } else if (op.isa()) { + vec_instruction_base_.emplace_back( + std::make_unique(op_idx++, + place_, + &op, + scope_, + local_scope_, + value_exe_info_.get())); } else { PADDLE_THROW(platform::errors::Unimplemented( "Now only support pd_kernel and cinn dialect.")); } - } else if (op->dialect()->name() == "pd_kernel") { - auto op_name = op->attributes() + } else if (op.dialect()->name() == "pd_kernel") { + auto op_name = op.attributes() .at("op_name") .dyn_cast<::pir::StrAttribute>() .AsString(); @@ -667,19 +672,19 @@ void PirInterpreter::BuildInstruction() { } VLOG(6) << "process " << op_name; - if (op->isa()) { + if (op.isa()) { vec_instruction_base_.emplace_back( std::make_unique( - op_idx++, place_, op, *(value_exe_info_.get()))); + op_idx++, place_, &op, *(value_exe_info_.get()))); } else { vec_instruction_base_.emplace_back( std::make_unique( - op_idx++, place_, op, *(value_exe_info_.get()))); + op_idx++, place_, &op, *(value_exe_info_.get()))); } #ifdef PADDLE_WITH_CINN - } else if (op->dialect()->name() == "cinn_runtime") { + } else if (op.dialect()->name() == "cinn_runtime") { vec_instruction_base_.emplace_back(std::make_unique( - op_idx++, place_, op, *(value_exe_info_.get()))); + op_idx++, place_, &op, *(value_exe_info_.get()))); #endif } else { PADDLE_THROW(platform::errors::Unimplemented( diff --git a/paddle/fluid/framework/new_executor/standalone_executor.cc b/paddle/fluid/framework/new_executor/standalone_executor.cc index aa97dab1fefe52..db1e522ad636fd 100644 --- a/paddle/fluid/framework/new_executor/standalone_executor.cc +++ b/paddle/fluid/framework/new_executor/standalone_executor.cc @@ -83,19 +83,15 @@ StandaloneExecutor::StandaloneExecutor(const platform::Place& place, std::shared_ptr<::pir::Program> base_program = ir_program; auto block = base_program->block(); for (auto it = block->begin(); it != block->end(); ++it) { - if ((*it)->isa()) { - size_t index = (*it) - ->attributes() - .at("col") - .dyn_cast() - .data(); + if (it->isa()) { + size_t index = + it->attributes().at("col").dyn_cast().data(); if (fetch_var_names_.size() < index + 1) { fetch_var_names_.resize(index + 1); } - fetch_var_names_[index] = (*it) - ->attributes() + fetch_var_names_[index] = it->attributes() .at("name") .dyn_cast() .AsString() + diff --git a/paddle/fluid/ir_adaptor/translator/program_translator.cc b/paddle/fluid/ir_adaptor/translator/program_translator.cc index 06d50c6b8fa9e3..f2bf300f2a40da 100644 --- a/paddle/fluid/ir_adaptor/translator/program_translator.cc +++ b/paddle/fluid/ir_adaptor/translator/program_translator.cc @@ -693,7 +693,7 @@ void ProgramTranslator::SetParameterFromSingleBlock(const BlockDesc& block) { pir::Block* block = program_->block(); pir::Block::Iterator insert_pos = std::find( - block->begin(), block->end(), defining_op_result.owner()); + block->begin(), block->end(), *defining_op_result.owner()); IR_ENFORCE( insert_pos != block->end(), diff --git a/paddle/fluid/pir/dialect/operator/ir/control_flow_op.cc b/paddle/fluid/pir/dialect/operator/ir/control_flow_op.cc index b185c9c3a05595..a76695a1012918 100644 --- a/paddle/fluid/pir/dialect/operator/ir/control_flow_op.cc +++ b/paddle/fluid/pir/dialect/operator/ir/control_flow_op.cc @@ -101,14 +101,14 @@ void IfOp::Print(pir::IrPrinter &printer) { os << " -> "; printer.PrintOpReturnType(op); os << "{"; - for (auto item : *true_block()) { + for (auto &item : *true_block()) { os << "\n "; - printer.PrintOperation(item); + printer.PrintOperation(&item); } os << "\n } else {"; - for (auto item : *false_block()) { + for (auto &item : *false_block()) { os << "\n "; - printer.PrintOperation(item); + printer.PrintOperation(&item); } os << "\n }"; } @@ -218,9 +218,9 @@ void WhileOp::Print(pir::IrPrinter &printer) { body_block()->args_end(), [&](pir::Value v) { printer.PrintValue(v); }, [&]() { os << ", "; }); - for (auto item : *body_block()) { + for (auto &item : *body_block()) { os << "\n "; - printer.PrintOperation(item); + printer.PrintOperation(&item); } os << "\n }"; } diff --git a/paddle/fluid/pir/drr/drr_rewrite_pattern.cc b/paddle/fluid/pir/drr/drr_rewrite_pattern.cc index 811bada792ae03..2da146c5dccbb6 100644 --- a/paddle/fluid/pir/drr/drr_rewrite_pattern.cc +++ b/paddle/fluid/pir/drr/drr_rewrite_pattern.cc @@ -369,9 +369,9 @@ MatchContextImpl DrrRewritePattern::CreateOperations( std::vector> temp_program; std::unordered_map op_2_temp_program_index; - for (Operation* op : *rewriter.block()) { - op_2_temp_program_index[op] = temp_program.size(); - temp_program.push_back({op}); + for (auto& op : *rewriter.block()) { + op_2_temp_program_index[&op] = temp_program.size(); + temp_program.push_back({&op}); } // topo order visit result_pattern_graph diff --git a/paddle/fluid/pir/transforms/build_cinn_pass.cc b/paddle/fluid/pir/transforms/build_cinn_pass.cc index 866c7327f9783d..c6f5f927895108 100644 --- a/paddle/fluid/pir/transforms/build_cinn_pass.cc +++ b/paddle/fluid/pir/transforms/build_cinn_pass.cc @@ -184,11 +184,11 @@ std::vector InverselyTopologicalSort(pir::Block* block) { std::vector sort_ops; std::unordered_map pending_count; // step 1: initialize pending_cout for defined op - for (auto* op : *block) { - if (pending_count.find(op) == pending_count.end()) { - pending_count[op] = 0; + for (auto& op : *block) { + if (pending_count.find(&op) == pending_count.end()) { + pending_count[&op] = 0; } - for (auto& operand : op->operands()) { + for (auto operand : op.operands()) { if (!operand || !(operand.source())) { continue; } @@ -202,10 +202,10 @@ std::vector InverselyTopologicalSort(pir::Block* block) { } std::queue queue; - for (auto* op : *block) { - VLOG(4) << op->name() << " pending_count: " << pending_count[op]; - if (pending_count[op] == 0) { - queue.push(op); + for (auto& op : *block) { + VLOG(4) << op.name() << " pending_count: " << pending_count[&op]; + if (pending_count[&op] == 0) { + queue.push(&op); } } @@ -328,8 +328,8 @@ class CinnSubgraphDetector { : block_(block), op_classifier_(classifier) { sort_ops_ = InverselyTopologicalSort(block_); size_t index = 0; - for (auto* op : *block) { - op2id_[op] = index++; + for (auto& op : *block) { + op2id_[&op] = index++; } } @@ -649,7 +649,7 @@ void ReplaceWithGroupOp(pir::Block* block, auto new_group_op = builder.Build(output_types); pir::Block* group_block = new_group_op.block(); - for (auto* op : group_ops) { + for (auto op : group_ops) { op->MoveTo(group_block, group_block->begin()); } diff --git a/paddle/fluid/pir/transforms/inplace_pass.cc b/paddle/fluid/pir/transforms/inplace_pass.cc index 07710bbaae2bb3..c008312af74dcc 100644 --- a/paddle/fluid/pir/transforms/inplace_pass.cc +++ b/paddle/fluid/pir/transforms/inplace_pass.cc @@ -144,22 +144,22 @@ static bool IsNoNeedBuffer(pir::Operation* op, pir::Value value) { static std::unordered_set GetSkipDeletionValues(pir::Block* block) { std::unordered_set skip_dels; for (auto& op : *block) { - if (op->dialect()->name().compare(paddle::dialect::KernelDialect::name()) != + if (op.dialect()->name().compare(paddle::dialect::KernelDialect::name()) != 0) { continue; } - IR_ENFORCE(op->attributes().count("op_name") > 0, + IR_ENFORCE(op.attributes().count("op_name") > 0, "kernel_dialect op should own an 'op_name' attribute."); auto upper_op_name = - op->attributes().at("op_name").dyn_cast().AsString(); + op.attributes().at("op_name").dyn_cast().AsString(); if (upper_op_name == "pd_op.feed" || upper_op_name == "pd_op.data") { - skip_dels.insert(op->result(0)); + skip_dels.insert(op.result(0)); continue; } if (upper_op_name == "pd_op.fetch" || upper_op_name == "builtin.shadow_output") { - skip_dels.insert(op->operand_source(0)); + skip_dels.insert(op.operand_source(0)); continue; } } @@ -174,19 +174,19 @@ static void GetEagerDelValueOfOp( const std::unordered_set& skip_dels, std::unordered_map* del_value_2_op) { for (auto& op : *block) { - std::string upper_op_name = op->name(); - if (op->dialect()->name().compare(paddle::dialect::KernelDialect::name()) == + std::string upper_op_name = op.name(); + if (op.dialect()->name().compare(paddle::dialect::KernelDialect::name()) == 0) { - IR_ENFORCE(op->attributes().count("op_name") > 0, + IR_ENFORCE(op.attributes().count("op_name") > 0, "kernel_dialect op should own an 'op_name' attribute."); - upper_op_name = op->attributes() + upper_op_name = op.attributes() .at("op_name") .dyn_cast() .AsString(); } - for (size_t i = 0; i < op->num_operands(); ++i) { - auto input = op->operand_source(i); + for (size_t i = 0; i < op.num_operands(); ++i) { + auto input = op.operand_source(i); if (skip_dels.count(input) > 0 || !input || !CanBeDeleted(input)) { VLOG(6) << "The " << i << "-th input value of the Operation(" << upper_op_name << ") can not be deleted."; @@ -195,18 +195,18 @@ static void GetEagerDelValueOfOp( VLOG(8) << " -- can be deleted: " << !CanBeDeleted(input); continue; } - (*del_value_2_op)[input] = op; + (*del_value_2_op)[input] = &op; } - for (auto& result : op->results()) { + for (auto& result : op.results()) { pir::Value output = result; if (output && CanBeDeleted(output)) { - (*del_value_2_op)[output] = op; + (*del_value_2_op)[output] = &op; } } - if (op->isa()) { - auto if_op = op->dyn_cast(); + if (op.isa()) { + auto if_op = op.dyn_cast(); GetEagerDelValueOfOp(if_op.true_block(), skip_dels, del_value_2_op); VLOG(8) << "GetEagerDelValueOfOp for IfOp true block"; GetEagerDelValueOfOp(if_op.false_block(), skip_dels, del_value_2_op); @@ -242,22 +242,22 @@ static std::unordered_map GetInplaceOps( std::unordered_set reused_output_values; for (auto& op : *block) { - for (size_t i = 0; i < op->num_operands(); ++i) { - visited_values.insert(op->operand_source(i)); + for (size_t i = 0; i < op.num_operands(); ++i) { + visited_values.insert(op.operand_source(i)); } - if (op->dialect()->name().compare(paddle::dialect::KernelDialect::name()) != + if (op.dialect()->name().compare(paddle::dialect::KernelDialect::name()) != 0) { - VLOG(6) << op->name() + VLOG(6) << op.name() << "is not a kernel_dialect op, inplace only support " "kernel_dialect operators"; - for (auto& result : op->results()) { + for (auto& result : op.results()) { visited_values.insert(result); } continue; } - auto upper_op_attrs = op->attributes(); + auto upper_op_attrs = op.attributes(); auto upper_op_name = upper_op_attrs.at("op_name").dyn_cast().AsString(); VLOG(6) << "analyse op: " << upper_op_name; @@ -271,7 +271,7 @@ static std::unordered_map GetInplaceOps( .dyn_cast() .data() .backend() == phi::Backend::CPU)) { - for (auto& result : op->results()) { + for (auto& result : op.results()) { visited_values.insert(result); } continue; @@ -280,10 +280,10 @@ static std::unordered_map GetInplaceOps( if (upper_op_attrs.count("is_inplace") != 0 && upper_op_attrs.at("is_inplace").dyn_cast().data()) { VLOG(6) << upper_op_name << " is already an inplace op."; - for (size_t i = 0; i < op->num_operands(); ++i) { - reused_input_values.insert(op->operand_source(i)); + for (size_t i = 0; i < op.num_operands(); ++i) { + reused_input_values.insert(op.operand_source(i)); } - for (auto& result : op->results()) { + for (auto& result : op.results()) { reused_output_values.insert(result); visited_values.insert(result); } @@ -306,12 +306,12 @@ static std::unordered_map GetInplaceOps( VLOG(6) << upper_op_name << "'s value can't delete or doesn't have inplace op, so that " "can't do inplace."; - for (size_t i = 0; i < op->num_results(); ++i) { - visited_values.insert(op->result(i)); + for (size_t i = 0; i < op.num_results(); ++i) { + visited_values.insert(op.result(i)); } continue; } - if (eager_dels.count(op) == 0 || (!upper_inplace_op_info) || + if (eager_dels.count(&op) == 0 || (!upper_inplace_op_info) || upper_op_name == "pd_op.transpose") { // NOTE(wanghuancoder): pd_op.transpose is not an // inplace op, only strided transpose support @@ -319,7 +319,7 @@ static std::unordered_map GetInplaceOps( VLOG(6) << upper_op_name << "'s value can't delete or doesn't have inplace op, so that " "can't do inplace."; - for (auto& result : op->results()) { + for (auto& result : op.results()) { visited_values.insert(result); } continue; @@ -341,49 +341,48 @@ static std::unordered_map GetInplaceOps( for (auto& kv : inplace_out_2_in) { uint32_t out_slot = kv.first; uint32_t in_slot = kv.second; - if ((in_slot >= op->num_operands()) || (out_slot >= op->num_results()) || - (!CanDoInplace(eager_dels.at(op), - op->operand_source(in_slot), - op->result(out_slot))) || - (visited_values.count(op->result(out_slot)) > 0) || - (!CanBeDeleted(op->result(out_slot))) || - (reused_input_values.count(op->operand_source(in_slot)) > 0) || - (reused_output_values.count(op->result(out_slot)) > 0)) { + if ((in_slot >= op.num_operands()) || (out_slot >= op.num_results()) || + (!CanDoInplace(eager_dels.at(&op), + op.operand_source(in_slot), + op.result(out_slot))) || + (visited_values.count(op.result(out_slot)) > 0) || + (!CanBeDeleted(op.result(out_slot))) || + (reused_input_values.count(op.operand_source(in_slot)) > 0) || + (reused_output_values.count(op.result(out_slot)) > 0)) { can_do_inplace = false; VLOG(6) << upper_op_name << "'s value has been visited or reused by other inplace op, " "so that can't do inplace."; VLOG_IF( - 8, - ((in_slot < op->num_operands()) && (out_slot < op->num_results()))) + 8, ((in_slot < op.num_operands()) && (out_slot < op.num_results()))) << " -- operand " << in_slot << " and result " << out_slot << " can do inplace: " - << CanDoInplace(eager_dels.at(op), - op->operand_source(in_slot), - op->result(out_slot)); - VLOG_IF(8, out_slot < op->num_results()) + << CanDoInplace(eager_dels.at(&op), + op.operand_source(in_slot), + op.result(out_slot)); + VLOG_IF(8, out_slot < op.num_results()) << " -- result " << out_slot - << " visited: " << (visited_values.count(op->result(out_slot)) > 0); - VLOG_IF(8, in_slot < op->num_operands()) + << " visited: " << (visited_values.count(op.result(out_slot)) > 0); + VLOG_IF(8, in_slot < op.num_operands()) << " -- operand " << in_slot << " has been reused: " - << (reused_input_values.count(op->operand_source(in_slot)) > 0); - VLOG_IF(8, out_slot < op->num_results()) + << (reused_input_values.count(op.operand_source(in_slot)) > 0); + VLOG_IF(8, out_slot < op.num_results()) << " -- result " << out_slot << " has been reused: " - << (reused_output_values.count(op->result(out_slot)) > 0); + << (reused_output_values.count(op.result(out_slot)) > 0); break; } } if (can_do_inplace) { - inplace_ops[op] = upper_op_name + "_"; + inplace_ops[&op] = upper_op_name + "_"; for (auto& kv : inplace_out_2_in) { - reused_input_values.insert(op->operand_source(kv.second)); - reused_output_values.insert(op->result(kv.first)); + reused_input_values.insert(op.operand_source(kv.second)); + reused_output_values.insert(op.result(kv.first)); } VLOG(6) << upper_op_name << " will change to inplace version op: " << upper_op_name + "_"; } - for (auto& result : op->results()) { + for (auto& result : op.results()) { visited_values.insert(result); } } @@ -414,7 +413,7 @@ class InplacePass : public pir::Pass { .dyn_cast() .AsString(); pir::Block::Iterator insert_pos = - std::find(block->begin(), block->end(), kv.first); + std::find(block->begin(), block->end(), *kv.first); IR_ENFORCE(insert_pos != block->end(), "Operator %s not found in block.", kv.first->name()); diff --git a/paddle/fluid/pir/transforms/params_sync_among_devices_pass.cc b/paddle/fluid/pir/transforms/params_sync_among_devices_pass.cc index 1fabb1d542016c..1f7e7704f20ef0 100644 --- a/paddle/fluid/pir/transforms/params_sync_among_devices_pass.cc +++ b/paddle/fluid/pir/transforms/params_sync_among_devices_pass.cc @@ -45,16 +45,16 @@ class ParamsSyncAmongDevicesPass : public pir::Pass { phi::errors::PreconditionNotMet( "params_sync_among_devices_pass should run on module op.")); auto* block = module_op.block(); - for (auto& op : *block) { - if (op->attributes().count("op_name") == 0) { + for (auto& inner_op : *block) { + if (inner_op.attributes().count("op_name") == 0) { continue; } - auto op_name = op->attributes() + auto op_name = inner_op.attributes() .at("op_name") .dyn_cast() .AsString(); if (op_name == pir::GetParameterOp::name()) { - auto use_op = pir::GetUseOpsForOutput(op, 0).front(); + auto use_op = pir::GetUseOpsForOutput(&inner_op, 0).front(); phi::KernelKey kernel_key; if (use_op->attributes().count("kernel_key")) { kernel_key = use_op->attributes() @@ -65,7 +65,7 @@ class ParamsSyncAmongDevicesPass : public pir::Pass { // TODO(liuyuanle): When the kernel_key doesn't exist？ if (use_op->attributes().count("kernel_key") && kernel_key.backend() != phi::Backend::CPU) { - std::string param_name = op->attributes() + std::string param_name = inner_op.attributes() .at("parameter_name") .dyn_cast() .AsString(); diff --git a/paddle/fluid/pir/transforms/pd_op_to_kernel_pass.cc b/paddle/fluid/pir/transforms/pd_op_to_kernel_pass.cc index e10b5898f2f5d3..495ac2602ce324 100644 --- a/paddle/fluid/pir/transforms/pd_op_to_kernel_pass.cc +++ b/paddle/fluid/pir/transforms/pd_op_to_kernel_pass.cc @@ -151,9 +151,9 @@ static phi::Backend ChooseInputBackend(const phi::Kernel& kernel, static std::set GetInputsByDataOp(pir::Block* block) { std::set data_op_names; - for (auto op_item : *block) { - if (op_item->isa()) { - data_op_names.insert(op_item->attributes() + for (auto& op_item : *block) { + if (op_item.isa()) { + data_op_names.insert(op_item.attributes() .at("name") .dyn_cast() .AsString()); @@ -1606,10 +1606,10 @@ void ProcessBlock( std::unordered_map* map_value_pair) { auto inputs_by_data_op = GetInputsByDataOp(block); - for (auto op_item : *block) { - VLOG(6) << "op name " << op_item->name(); - if ((op_item->isa()) && - inputs_by_data_op.count(op_item->attributes() + for (auto& op_item : *block) { + VLOG(6) << "op name " << op_item.name(); + if ((op_item.isa()) && + inputs_by_data_op.count(op_item.attributes() .at("name") .dyn_cast() .AsString())) { @@ -1618,24 +1618,24 @@ void ProcessBlock( } // HandleSpecialOp - if (SpecialLowerOps.count(op_item->name())) { - VLOG(6) << "Handle Special Op: [" << op_item->name() + if (SpecialLowerOps.count(op_item.name())) { + VLOG(6) << "Handle Special Op: [" << op_item.name() << "] while lowering to kernel pass"; HandleForSpecialOp( - place, op_item, new_block, ctx, map_op_pair, map_value_pair); + place, &op_item, new_block, ctx, map_op_pair, map_value_pair); continue; } - auto op_info_parser = GetOpYamlInfoParser(op_item); - auto kernel_name = GetKernelName(op_info_parser.get(), op_item); + auto op_info_parser = GetOpYamlInfoParser(&op_item); + auto kernel_name = GetKernelName(op_info_parser.get(), &op_item); auto kernel_key = GetKernelKey( - op_item, place, kernel_name, *map_value_pair, op_info_parser.get()); + &op_item, place, kernel_name, *map_value_pair, op_info_parser.get()); VLOG(6) << "kernel type " << kernel_key; // build output type - auto op_output_types = BuildOutputs(op_item, kernel_name, kernel_key, ctx); + auto op_output_types = BuildOutputs(&op_item, kernel_name, kernel_key, ctx); // build input - auto vec_inputs = BuildInputs(op_item, + auto vec_inputs = BuildInputs(&op_item, kernel_name, kernel_key, place, @@ -1650,14 +1650,14 @@ void ProcessBlock( kernel_key, vec_inputs, op_output_types, - op_item, + &op_item, new_block, ctx, map_op_pair, map_value_pair); AddShadowFeedOpForDataOrFeed( - place, op_item, op, new_block, ctx, map_op_pair, map_value_pair); + place, &op_item, op, new_block, ctx, map_op_pair, map_value_pair); } } diff --git a/paddle/fluid/pybind/control_flow_api.cc b/paddle/fluid/pybind/control_flow_api.cc index cc3069ccf88ecb..5f7ca7f142cdaa 100644 --- a/paddle/fluid/pybind/control_flow_api.cc +++ b/paddle/fluid/pybind/control_flow_api.cc @@ -19,26 +19,38 @@ #include #include #include +#include +#include #include "paddle/fluid/pir/dialect/operator/ir/api_builder.h" +#include "paddle/fluid/pir/dialect/operator/ir/control_flow_op.h" #include "paddle/fluid/platform/enforce.h" +#include "paddle/phi/common/data_type.h" +#include "paddle/phi/common/place.h" +#include "paddle/pir/core/op_result.h" +#include "paddle/pir/core/operation.h" #include "paddle/pir/dialect/control_flow/ir/cf_op.h" namespace py = pybind11; using paddle::dialect::ApiBuilder; +using paddle::dialect::IfOp; +using pir::Block; +using pir::Builder; +using pir::Operation; +using pir::Region; +using pir::Type; +using pir::Value; using pir::YieldOp; using pybind11::return_value_policy; -namespace paddle { -namespace pybind { - -class PyIfOp : public dialect::IfOp { +namespace { +class PyIfOp : public IfOp { public: - explicit PyIfOp(dialect::IfOp if_op); + explicit PyIfOp(IfOp if_op); void UpdateOutput(); }; -PyIfOp::PyIfOp(dialect::IfOp if_op) : IfOp(if_op) { +PyIfOp::PyIfOp(IfOp if_op) : IfOp(if_op) { PADDLE_ENFORCE_NOT_NULL( if_op, paddle::platform::errors::InvalidArgument( @@ -50,32 +62,31 @@ void PyIfOp::UpdateOutput() { *this, paddle::platform::errors::InvalidArgument( "The if_op in PyIfOp used to update output can't be nullptr")); - auto block = (*this)->GetParent(); + auto block = parent(); PADDLE_ENFORCE_NOT_NULL(block, paddle::platform::errors::InvalidArgument( "The parent block of if_op which used to update " "output can't be nullptr")); - pir::Block::Iterator iter = **this; - pir::Builder builder(ir_context(), false); - auto new_if_op = builder.Build( + Block::Iterator iter = **this; + Builder builder(ir_context(), false); + auto new_if_op = builder.Build( cond(), true_region().TakeBack(), false_region().TakeBack()); block->Assign(iter, new_if_op); IfOp::operator=(new_if_op); VerifyRegion(); } -PyIfOp BuildPyIfOp(pir::Value cond) { - return PyIfOp( - dialect::ApiBuilder::Instance().GetBuilder()->Build( - cond, std::vector{})); +PyIfOp BuildPyIfOp(Value cond) { + return PyIfOp(ApiBuilder::Instance().GetBuilder()->Build( + cond, std::vector{})); } -void BindIfOp(py::module *m) { +void BindIfOp(py::module* m) { m->def("build_if_op", BuildPyIfOp); m->def("cf_yield", [](py::list inputs) { - std::vector input_values; + std::vector input_values; for (auto input : inputs) { - input_values.push_back(input.cast()); + input_values.push_back(input.cast()); } ApiBuilder::Instance().GetBuilder()->Build(input_values); }); @@ -87,7 +98,7 @@ void BindIfOp(py::module *m) { if_op.def("true_block", &PyIfOp::true_block, return_value_policy::reference) .def("false_block", &PyIfOp::false_block, return_value_policy::reference) .def("update_output", &PyIfOp::UpdateOutput) - .def("results", [](PyIfOp &self) -> py::list { + .def("results", [](PyIfOp& self) -> py::list { py::list op_list; for (uint32_t i = 0; i < self->num_results(); i++) { op_list.append(self.result(i)); @@ -95,6 +106,49 @@ void BindIfOp(py::module *m) { return op_list; }); } -void BindControlFlowApi(py::module *m) { BindIfOp(m); } + +void GetUsedExternalValueImpl( + std::unordered_set& defined_values, // NOLINT + std::vector& used_values, // NOLINT + const Operation& op) { + for (size_t index = 0; index < op.num_operands(); ++index) { + Value value = op.operand_source(index); + if (defined_values.find(value) == defined_values.end()) { + used_values.push_back(value); + defined_values.insert(value); + } + } + for (auto& region : op) { + for (auto& block : region) { + for (auto value : block.args()) { + defined_values.insert(value); + } + } + for (auto& block : region) { + for (auto& inner_op : block) { + GetUsedExternalValueImpl(defined_values, used_values, inner_op); + } + } + } + for (size_t index = 0; index < op.num_results(); ++index) { + defined_values.insert(op.result(index)); + } +} + +std::vector GetUsedExternalValue(const Operation& op) { + std::unordered_set defined_values{nullptr}; + std::vector used_values; + GetUsedExternalValueImpl(defined_values, used_values, op); + return used_values; +} + +} // namespace + +namespace paddle { +namespace pybind { +void BindControlFlowApi(py::module* m) { + m->def("get_used_external_value", GetUsedExternalValue); + BindIfOp(m); +} } // namespace pybind } // namespace paddle diff --git a/paddle/fluid/pybind/control_flow_api.h b/paddle/fluid/pybind/control_flow_api.h index 29f2950a01ac20..65df07d9f5e06d 100644 --- a/paddle/fluid/pybind/control_flow_api.h +++ b/paddle/fluid/pybind/control_flow_api.h @@ -15,12 +15,6 @@ #pragma once #include -#include - -#include "paddle/fluid/pir/dialect/operator/ir/control_flow_op.h" -#include "paddle/phi/common/data_type.h" -#include "paddle/phi/common/place.h" -#include "paddle/pir/core/op_result.h" namespace paddle { namespace pybind { diff --git a/paddle/fluid/pybind/pir.cc b/paddle/fluid/pybind/pir.cc index 41de635ce9a551..4e8e954a312a85 100644 --- a/paddle/fluid/pybind/pir.cc +++ b/paddle/fluid/pybind/pir.cc @@ -270,15 +270,14 @@ void BindBlock(py::module *m) { "program", [](Block &self) { return self.GetParentOp()->GetParentProgram(); }, return_value_policy::reference) - .def_property_readonly( - "ops", - [](Block &self) -> py::list { - py::list op_list; - for (auto iter = self.begin(); iter != self.end(); iter++) { - op_list.append(*iter); - } - return op_list; - }) + .def_property_readonly("ops", + [](Block &self) -> py::list { + py::list op_list; + for (auto &op : self) { + op_list.append(&op); + } + return op_list; + }) .def("__enter__", [](Block &self) { ApiBuilder::Instance().PushInsertionPoint({&self, self.end()}); @@ -290,7 +289,7 @@ void BindBlock(py::module *m) { .def( "remove_op", [](Block &self, Operation *op) { - auto op_iter = std::find(self.begin(), self.end(), op); + auto op_iter = std::find(self.begin(), self.end(), *op); self.erase(op_iter); }, R"DOC( @@ -324,17 +323,16 @@ void BindBlock(py::module *m) { .def("all_parameters", [](Block &self) -> py::list { py::list param_list; - for (auto iter = self.begin(); iter != self.end(); iter++) { - auto op = *iter; - if (op->HasAttribute(kAttrIsPersisable)) { - auto attrs = op->attribute(kAttrIsPersisable) + for (auto &op : self) { + if (op.HasAttribute(kAttrIsPersisable)) { + auto attrs = op.attribute(kAttrIsPersisable) .dyn_cast() .AsVector(); for (uint32_t i = 0; i < attrs.size(); i++) { bool is_persistable = attrs[i].dyn_cast().data(); if (is_persistable) { - param_list.append(op->result(i)); + param_list.append(op.result(i)); } } } @@ -342,8 +340,8 @@ void BindBlock(py::module *m) { return param_list; }) .def("refresh_stopgradient", [](Block &self) { - for (auto iter = self.begin(); iter != self.end(); iter++) { - RefreshOpStopgradients(*iter); + for (auto &op : self) { + RefreshOpStopgradients(&op); } }); } @@ -1038,7 +1036,7 @@ std::pair, OpResultMap> CloneProgram( auto cloned_program = std::make_shared(ctx); std::unordered_map value_map; for (auto &op : *program.block()) { - auto *cloned_op = BuildOpFrom(op, value_map); + auto *cloned_op = BuildOpFrom(&op, value_map); cloned_program->block()->push_back(cloned_op); } std::unordered_map op_result_map; @@ -1203,17 +1201,16 @@ SplitedResult SplitForwardBackward( for (auto it = forward_program->block()->rbegin(); it != forward_program->block()->rend(); ++it) { - auto *op = *it; - if (op->isa()) { + if (it->isa()) { auto out_name = - op->attribute("parameter_name").AsString(); + it->attribute("parameter_name").AsString(); if (out_name == parameter_name) { VLOG(4) << out_name << " has been inserted SetParameterOp, skip it now."; return; } - inserted_value.insert(op->operand_source(0)); + inserted_value.insert(it->operand_source(0)); } } diff --git a/paddle/pir/core/block.cc b/paddle/pir/core/block.cc index 56331bc7546f04..e2bc4d5bcf12ef 100644 --- a/paddle/pir/core/block.cc +++ b/paddle/pir/core/block.cc @@ -43,8 +43,8 @@ Block::Iterator Block::insert(ConstIterator iterator, Operation *op) { } Block::Iterator Block::erase(ConstIterator position) { - IR_ENFORCE((*position)->GetParent() == this, "iterator not own this block."); - (*position)->Destroy(); + IR_ENFORCE(position->GetParent() == this, "iterator not own this block."); + position->Destroy(); return ops_.erase(position); } @@ -56,19 +56,19 @@ void Block::clear() { } void Block::Assign(Iterator position, Operation *op) { - IR_ENFORCE((*position)->GetParent() == this, "position not own this block."); - (*position)->Destroy(); - (*position) = op; + IR_ENFORCE(position->GetParent() == this, "position not own this block."); + position->Destroy(); + position.set_underlying_pointer(op); op->SetParent(this, position); } Operation *Block::Take(Operation *op) { IR_ENFORCE(op && op->GetParent() == this, "iterator not own this block."); - ops_.erase(*op); + ops_.erase(Iterator(*op)); return op; } -void Block::SetParent(Region *parent, Region::iterator position) { +void Block::SetParent(Region *parent, Region::Iterator position) { parent_ = parent; position_ = position; } diff --git a/paddle/pir/core/block.h b/paddle/pir/core/block.h index 46dd412d5b0360..7a755b33c2e02e 100644 --- a/paddle/pir/core/block.h +++ b/paddle/pir/core/block.h @@ -20,8 +20,8 @@ #include "paddle/pir/core/block_argument.h" #include "paddle/pir/core/block_operand.h" #include "paddle/pir/core/dll_decl.h" +#include "paddle/pir/core/iterator.h" #include "paddle/pir/core/region.h" -#include "paddle/pir/core/use_iterator.h" namespace pir { class Operation; @@ -30,9 +30,11 @@ class IR_API Block { using OpListType = std::list; public: - using Iterator = OpListType::iterator; - using ReverseIterator = OpListType::reverse_iterator; - using ConstIterator = OpListType::const_iterator; + using Iterator = PointerListIterator; + using ConstIterator = PointerListConstIterator; + + using ReverseIterator = std::reverse_iterator; + using ConstReverseIterator = std::reverse_iterator; Block() = default; ~Block(); @@ -47,6 +49,8 @@ class IR_API Block { ConstIterator end() const { return ops_.end(); } Iterator begin() { return ops_.begin(); } Iterator end() { return ops_.end(); } + ConstReverseIterator rbegin() const { return ops_.rbegin(); } + ConstReverseIterator rend() const { return ops_.rend(); } ReverseIterator rbegin() { return ops_.rbegin(); } ReverseIterator rend() { return ops_.rend(); } @@ -57,7 +61,7 @@ class IR_API Block { Iterator insert(ConstIterator iterator, Operation *op); Iterator erase(ConstIterator position); void clear(); - operator Region::iterator() { return position_; } + operator Region::Iterator() { return position_; } // Assign the operation underlying in position with parameter op, // meanwhile, destroy the original operation. @@ -83,9 +87,12 @@ class IR_API Block { /// using BlockArgListType = std::vector; using ArgsIterator = BlockArgListType::iterator; + using ConstArgsIterator = BlockArgListType::const_iterator; ArgsIterator args_begin() { return arguments_.begin(); } ArgsIterator args_end() { return arguments_.end(); } + ConstArgsIterator args_begin() const { return arguments_.begin(); } + ConstArgsIterator args_end() const { return arguments_.end(); } bool args_empty() const { return arguments_.empty(); } uint32_t args_size() const { return arguments_.size(); } const BlockArgListType &args() const { return arguments_; } @@ -109,7 +116,7 @@ class IR_API Block { // Allow access to 'SetParent'. friend class Region; - void SetParent(Region *parent, Region::iterator position); + void SetParent(Region *parent, Region::Iterator position); // Take out corresponding Operation and its ownershipe. friend class Operation; @@ -118,7 +125,7 @@ class IR_API Block { static bool TopoOrderCheck(const OpListType &op_list); private: - Region::iterator position_; + Region::Iterator position_; BlockOperand first_use_; OpListType ops_; // owned BlockArgListType arguments_; // owned diff --git a/paddle/pir/core/ir_printer.cc b/paddle/pir/core/ir_printer.cc index 37c74111d00e1e..0dfa960f183291 100644 --- a/paddle/pir/core/ir_printer.cc +++ b/paddle/pir/core/ir_printer.cc @@ -181,15 +181,15 @@ void IrPrinter::PrintFullOperation(Operation* op) { } void IrPrinter::PrintRegion(const Region& region) { - for (auto block : region) { + for (auto& block : region) { PrintBlock(block); } } -void IrPrinter::PrintBlock(const Block* block) { +void IrPrinter::PrintBlock(const Block& block) { os << "{\n"; - for (auto item : *block) { - PrintOperation(item); + for (auto& item : block) { + PrintOperation(&item); os << newline; } os << "}\n"; diff --git a/paddle/pir/core/ir_printer.h b/paddle/pir/core/ir_printer.h index cb7135fb484dea..858fe5dfc79542 100644 --- a/paddle/pir/core/ir_printer.h +++ b/paddle/pir/core/ir_printer.h @@ -56,7 +56,7 @@ class IR_API IrPrinter : public BasicIrPrinter { void PrintFullOperation(Operation* op); void PrintRegion(const Region& Region); - void PrintBlock(const Block* block); + void PrintBlock(const Block& block); void PrintValue(Value v); diff --git a/paddle/pir/core/iterator.h b/paddle/pir/core/iterator.h new file mode 100644 index 00000000000000..ce71b912b6de90 --- /dev/null +++ b/paddle/pir/core/iterator.h @@ -0,0 +1,190 @@ +// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once +#include +#include +namespace pir { + +class Operation; +/// +/// \brief Value Iterator +/// +template +class ValueUseIterator { + public: + ValueUseIterator(OperandType use = nullptr) : current_(use) {} // NOLINT + + bool operator==(const ValueUseIterator& rhs) const { + return current_ == rhs.current_; + } + bool operator!=(const ValueUseIterator& rhs) const { + return !(*this == rhs); + } + + Operation* owner() const { return current_.owner(); } + + OperandType& operator*() { return current_; } + + OperandType* operator->() { return &operator*(); } + + ValueUseIterator& operator++() { + current_ = current_.next_use(); + return *this; + } + + ValueUseIterator operator++(int) { + ValueUseIterator tmp = *this; + current_ = current_.next_use(); + return tmp; + } + + protected: + OperandType current_; +}; + +/// +/// \brief The wrapper for std::list::iterator +/// +template +class PointerListIterator { + typename std::list::iterator iterator_; + + public: + // use to support std::next, std::prev. std::advance + typedef ptrdiff_t difference_type; + typedef std::bidirectional_iterator_tag iterator_category; + typedef ElementType value_type; + typedef value_type* pointer; + typedef value_type& reference; + + PointerListIterator() = default; + PointerListIterator( + const typename std::list::iterator& iter) // NOLINT + : iterator_(iter) {} + + ElementType& operator*() const noexcept { return **iterator_; } + + ElementType* operator->() const noexcept { return &this->operator*(); } + + PointerListIterator& operator++() noexcept { + ++iterator_; + return *this; + } + PointerListIterator operator++(int) noexcept { + PointerListIterator __tmp = *this; + ++iterator_; + return __tmp; + } + + PointerListIterator& operator--() noexcept { + --iterator_; + return *this; + } + + PointerListIterator operator--(int) noexcept { + PointerListIterator __tmp = *this; + iterator_--; + return __tmp; + } + + bool operator==(const PointerListIterator& __x) const noexcept { + return iterator_ == __x.iterator_; + } + + bool operator!=(const PointerListIterator& __x) const noexcept { + return iterator_ != __x.iterator_; + } + + void set_underlying_pointer(ElementType* ptr) { *iterator_ = ptr; } + + operator typename std::list::iterator() const { + return iterator_; + } + operator typename std::list::const_iterator() const { + return iterator_; + } + + // If iterator do not point to a element, it is unsafe. + operator ElementType*() const { return *iterator_; } +}; + +/// +/// \brief The wrapper for std::list::const_iterator +/// +template +class PointerListConstIterator { + typename std::list::const_iterator iterator_; + + public: + // use to support std::next, std::prev. std::advance + typedef ptrdiff_t difference_type; + typedef std::bidirectional_iterator_tag iterator_category; + typedef ElementType value_type; + typedef value_type* pointer; + typedef value_type& reference; + + PointerListConstIterator() = default; + PointerListConstIterator( + const PointerListIterator& iter) // NOLINT + : iterator_( + static_cast::iterator>(iter)) {} + PointerListConstIterator( + const typename std::list::iterator& iter) + : iterator_(iter) {} // NOLINT + PointerListConstIterator( + const typename std::list::const_iterator& iter) + : iterator_(iter) {} // NOLINT + + ElementType& operator*() const noexcept { return **iterator_; } + + ElementType* operator->() const noexcept { return &this->operator*(); } + + PointerListConstIterator& operator++() noexcept { + ++iterator_; + return *this; + } + PointerListConstIterator operator++(int) noexcept { + PointerListConstIterator __tmp = *this; + ++iterator_; + return __tmp; + } + + PointerListConstIterator& operator--() noexcept { + --iterator_; + return *this; + } + + PointerListConstIterator operator--(int) noexcept { + PointerListConstIterator __tmp = *this; + iterator_--; + return __tmp; + } + + bool operator==(const PointerListConstIterator& __x) const noexcept { + return iterator_ == __x.iterator_; + } + + bool operator!=(const PointerListConstIterator& __x) const noexcept { + return iterator_ != __x.iterator_; + } + + operator typename std::list::const_iterator() const { + return iterator_; + } + // If iterator do not point to a element, it is unsafe. + operator ElementType*() const { return *iterator_; } +}; + +} // namespace pir diff --git a/paddle/pir/core/op_base.h b/paddle/pir/core/op_base.h index 5988e760678c43..d827a7afb476c3 100644 --- a/paddle/pir/core/op_base.h +++ b/paddle/pir/core/op_base.h @@ -24,6 +24,7 @@ namespace pir { class Builder; class IrPrinter; +class Block; class IR_API OpBase { public: @@ -46,6 +47,8 @@ class IR_API OpBase { uint32_t num_operands() const { return operation()->num_operands(); } + Block *parent() const { return operation()->GetParent(); } + const AttributeMap &attributes() const { return operation()->attributes(); } Value operand_source(uint32_t index) const { diff --git a/paddle/pir/core/operation.cc b/paddle/pir/core/operation.cc index 1a6666fcc2a9b3..a3fad429e75c05 100644 --- a/paddle/pir/core/operation.cc +++ b/paddle/pir/core/operation.cc @@ -354,17 +354,12 @@ int32_t Operation::ComputeOpOperandOffset(uint32_t index) const { sizeof(Operation)); } -#define COMPONENT_IMPL(component_lower, componnent_upper) \ - componnent_upper##Impl *Operation::component_lower##_impl(uint32_t index) { \ - int32_t offset = Compute##componnent_upper##Offset(index); \ - return reinterpret_cast( \ - reinterpret_cast(this) + offset); \ - } \ - const componnent_upper##Impl *Operation::component_lower##_impl( \ - uint32_t index) const { \ - int32_t offset = Compute##componnent_upper##Offset(index); \ - return reinterpret_cast( \ - reinterpret_cast(this) + offset); \ +#define COMPONENT_IMPL(component_lower, componnent_upper) \ + componnent_upper##Impl *Operation::component_lower##_impl(uint32_t index) \ + const { \ + int32_t offset = Compute##componnent_upper##Offset(index); \ + return reinterpret_cast( \ + reinterpret_cast(const_cast(this)) + offset); \ } COMPONENT_IMPL(op_result, OpResult) diff --git a/paddle/pir/core/operation.h b/paddle/pir/core/operation.h index a41e648e7e2793..b7abc5d8a07ea3 100644 --- a/paddle/pir/core/operation.h +++ b/paddle/pir/core/operation.h @@ -58,6 +58,8 @@ class IR_API alignas(8) Operation final { Dialect *dialect() const; + bool operator==(const Operation &other) const { return this == &other; } + /// /// \brief op attribute related public interfaces /// @@ -82,15 +84,15 @@ class IR_API alignas(8) Operation final { /// \brief op ouput related public interfaces /// uint32_t num_results() const { return num_results_; } - OpResult result(uint32_t index) { return op_result_impl(index); } - Type result_type(uint32_t index) { return result(index).type(); } + OpResult result(uint32_t index) const { return op_result_impl(index); } + Type result_type(uint32_t index) const { return result(index).type(); } std::vector results(); /// /// \brief op input related public interfaces /// uint32_t num_operands() const { return num_operands_; } - OpOperand operand(uint32_t index) { return op_operand_impl(index); } + OpOperand operand(uint32_t index) const { return op_operand_impl(index); } std::vector operands(); Value operand_source(uint32_t index) const; std::vector operands_source() const; @@ -107,9 +109,15 @@ class IR_API alignas(8) Operation final { /// /// \brief region related public interfaces /// + using Iterator = Region *; + using ConstIterator = const Region *; uint32_t num_regions() const { return num_regions_; } Region ®ion(unsigned index); const Region ®ion(unsigned index) const; + ConstIterator begin() const { return regions_; } + ConstIterator end() const { return regions_ + num_regions_; } + Iterator begin() { return regions_; } + Iterator end() { return regions_ + num_regions_; } /// /// \brief parent related public interfaces @@ -177,12 +185,10 @@ class IR_API alignas(8) Operation final { uint32_t num_successors); int32_t ComputeOpResultOffset(uint32_t index) const; - detail::OpResultImpl *op_result_impl(uint32_t index); - const detail::OpResultImpl *op_result_impl(uint32_t index) const; + detail::OpResultImpl *op_result_impl(uint32_t index) const; int32_t ComputeOpOperandOffset(uint32_t index) const; - detail::OpOperandImpl *op_operand_impl(uint32_t index); - const detail::OpOperandImpl *op_operand_impl(uint32_t index) const; + detail::OpOperandImpl *op_operand_impl(uint32_t index) const; template struct CastUtil { diff --git a/paddle/pir/core/region.cc b/paddle/pir/core/region.cc index 4e5a58e739dfef..7865d1f2158954 100644 --- a/paddle/pir/core/region.cc +++ b/paddle/pir/core/region.cc @@ -30,15 +30,15 @@ Block *Region::emplace_back() { void Region::push_front(Block *block) { insert(blocks_.begin(), block); } -Region::iterator Region::insert(const_iterator position, Block *block) { - Region::iterator iter = blocks_.insert(position, block); +Region::Iterator Region::insert(ConstIterator position, Block *block) { + Region::Iterator iter = blocks_.insert(position, block); block->SetParent(this, iter); return iter; } -Region::iterator Region::erase(const_iterator position) { - IR_ENFORCE((*position)->GetParent() == this, "iterator not own this region."); - delete *position; +Region::Iterator Region::erase(ConstIterator position) { + IR_ENFORCE(position->GetParent() == this, "iterator not own this region."); + delete position; return blocks_.erase(position); } diff --git a/paddle/pir/core/region.h b/paddle/pir/core/region.h index 9173025dc3c7f3..0fc62e985a3575 100644 --- a/paddle/pir/core/region.h +++ b/paddle/pir/core/region.h @@ -19,6 +19,7 @@ #include #include "paddle/pir/core/dll_decl.h" +#include "paddle/pir/core/iterator.h" namespace pir { @@ -28,9 +29,11 @@ class IrContext; class IR_API Region { public: - using iterator = std::list::iterator; - using reverse_iterator = std::list::reverse_iterator; - using const_iterator = std::list::const_iterator; + using Iterator = PointerListIterator; + using ConstIterator = PointerListConstIterator; + using ReverseIterator = std::reverse_iterator; + using ConstReverseIterator = std::reverse_iterator; + explicit Region(Operation *op = nullptr) : parent_(op) {} Region(const Region &) = delete; Region &operator=(const Region &) = delete; @@ -38,20 +41,22 @@ class IR_API Region { bool empty() const { return blocks_.empty(); } size_t size() const { return blocks_.size(); } - iterator begin() { return blocks_.begin(); } - iterator end() { return blocks_.end(); } - const_iterator begin() const { return blocks_.begin(); } - const_iterator end() const { return blocks_.end(); } - reverse_iterator rbegin() { return blocks_.rbegin(); } - reverse_iterator rend() { return blocks_.rend(); } + Iterator begin() { return blocks_.begin(); } + Iterator end() { return blocks_.end(); } + ConstIterator begin() const { return blocks_.begin(); } + ConstIterator end() const { return blocks_.end(); } + ReverseIterator rbegin() { return blocks_.rbegin(); } + ReverseIterator rend() { return blocks_.rend(); } + ConstReverseIterator rbegin() const { return blocks_.rbegin(); } + ConstReverseIterator rend() const { return blocks_.rend(); } Block *back() const { return blocks_.back(); } Block *front() const { return blocks_.front(); } void push_back(Block *block); Block *emplace_back(); void push_front(Block *block); - iterator insert(const_iterator position, Block *block); - iterator erase(const_iterator position); + Iterator insert(ConstIterator position, Block *block); + Iterator erase(ConstIterator position); void clear(); // take the last block of region. diff --git a/paddle/pir/core/use_iterator.h b/paddle/pir/core/use_iterator.h deleted file mode 100644 index 42705162d93e54..00000000000000 --- a/paddle/pir/core/use_iterator.h +++ /dev/null @@ -1,55 +0,0 @@ -// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#pragma once -namespace pir { - -class Operation; -/// -/// \brief Value Iterator -/// -template -class ValueUseIterator { - public: - ValueUseIterator(OperandType use = nullptr) : current_(use) {} // NOLINT - - bool operator==(const ValueUseIterator &rhs) const { - return current_ == rhs.current_; - } - bool operator!=(const ValueUseIterator &rhs) const { - return !(*this == rhs); - } - - Operation *owner() const { return current_.owner(); } - - OperandType &operator*() { return current_; } - - OperandType *operator->() { return &operator*(); } - - ValueUseIterator &operator++() { - current_ = current_.next_use(); - return *this; - } - - ValueUseIterator operator++(int) { - ValueUseIterator tmp = *this; - current_ = current_.next_use(); - return tmp; - } - - protected: - OperandType current_; -}; - -} // namespace pir diff --git a/paddle/pir/core/value.h b/paddle/pir/core/value.h index 50d8265f29884c..11d1193bbc0688 100644 --- a/paddle/pir/core/value.h +++ b/paddle/pir/core/value.h @@ -14,9 +14,9 @@ #pragma once +#include "paddle/pir/core/iterator.h" #include "paddle/pir/core/op_operand.h" #include "paddle/pir/core/type.h" -#include "paddle/pir/core/use_iterator.h" namespace pir { class Operation; diff --git a/paddle/pir/core/verify.cc b/paddle/pir/core/verify.cc index 2d3485324a6bab..40478d34952072 100644 --- a/paddle/pir/core/verify.cc +++ b/paddle/pir/core/verify.cc @@ -18,11 +18,10 @@ namespace pir { void Verify(Operation *op, bool verify_recursively) { op->Verify(); if (!verify_recursively) return; - for (size_t index = 0; index < op->num_regions(); ++index) { - auto ®ion = op->region(index); - for (auto block : region) { - for (auto op_item : *block) { - Verify(op_item, verify_recursively); + for (auto ®ion : *op) { + for (auto &block : region) { + for (auto &op_item : block) { + Verify(&op_item, verify_recursively); } } } diff --git a/paddle/pir/dialect/shape/ir/shape_op.cc b/paddle/pir/dialect/shape/ir/shape_op.cc index a9742a689f79dc..d8644bda1d07d8 100644 --- a/paddle/pir/dialect/shape/ir/shape_op.cc +++ b/paddle/pir/dialect/shape/ir/shape_op.cc @@ -251,9 +251,9 @@ void FuncOp::Print(IrPrinter &printer) { auto &os = printer.os; os << " shape.func () "; os << "{"; - for (auto item : *block()) { + for (auto &item : *block()) { os << "\n "; - printer.PrintOperation(item); + printer.PrintOperation(&item); } os << "\n }"; } diff --git a/paddle/pir/dialect/shape/transforms/shape_optimization.cc b/paddle/pir/dialect/shape/transforms/shape_optimization.cc index 28679ef4cc0153..11c38f2c89c1be 100644 --- a/paddle/pir/dialect/shape/transforms/shape_optimization.cc +++ b/paddle/pir/dialect/shape/transforms/shape_optimization.cc @@ -66,7 +66,7 @@ bool InsertTieShapeOnBlock(pir::Block* block) { // TODO(zhangbopd): mapping block arguments std::vector op_list; - for (pir::Operation* op : *block) op_list.push_back(op); + for (auto& op : *block) op_list.push_back(&op); for (pir::Operation* op : op_list) { if (!InsertTieShapeOnOperation(op, builder)) return false; } @@ -74,8 +74,8 @@ bool InsertTieShapeOnBlock(pir::Block* block) { } bool InsertTieShapeOnRegion(pir::Region* region) { - for (Block* block : *region) { - if (!InsertTieShapeOnBlock(block)) return false; + for (auto& block : *region) { + if (!InsertTieShapeOnBlock(&block)) return false; } return true; } @@ -241,8 +241,8 @@ bool ShapeComputationIRAnalysis::Run() { } bool ShapeComputationIRAnalysis::RunOnRegion(Region* region, func fn) { - for (Block* block : *region) { - if (!RunOnBlock(block, fn)) return false; + for (auto& block : *region) { + if (!RunOnBlock(&block, fn)) return false; } return true; } @@ -251,7 +251,7 @@ bool ShapeComputationIRAnalysis::RunOnBlock(Block* block, func fn) { // TODO(zhangbopd): mapping block arguments std::vector op_list; - for (Operation* op : *block) op_list.push_back(op); + for (auto& op : *block) op_list.push_back(&op); for (Operation* op : op_list) { if (!RunOnOperation(op, fn)) return false; } diff --git a/paddle/pir/dialect/shape/utils/shape_optimization_utils.cc b/paddle/pir/dialect/shape/utils/shape_optimization_utils.cc index 79f2ba55950588..b2594164713398 100644 --- a/paddle/pir/dialect/shape/utils/shape_optimization_utils.cc +++ b/paddle/pir/dialect/shape/utils/shape_optimization_utils.cc @@ -49,9 +49,9 @@ bool CompareSymbolicDimProduct(SymbolicDimProduct& lhs, // NOLINT } SymbolicDimMgr::SymbolicDimMgr(ModuleOp m) : m_(m) { - for (auto op : *(m.block())) { - if (op->isa()) { - symbol_table_ = SymbolTable(op); + for (auto& op : *(m.block())) { + if (op.isa()) { + symbol_table_ = SymbolTable(&op); return; } } @@ -63,9 +63,9 @@ SymbolicDimMgr::SymbolicDimMgr(ModuleOp m) : m_(m) { bool SymbolicDimMgr::Load() { auto func_op = symbol_table_.getOp()->dyn_cast(); IR_ENFORCE(func_op); - for (auto op : *(func_op.block())) { - symbol_table_.insert(op); - if (SymbolicDimOp sym_dim_op = op->dyn_cast()) { + for (auto& op : *(func_op.block())) { + symbol_table_.insert(&op); + if (SymbolicDimOp sym_dim_op = op.dyn_cast()) { symbol_dim_union_set_[sym_dim_op] = sym_dim_op; symbol_name_set_.insert(sym_dim_op.GetSymName()); } @@ -473,16 +473,16 @@ bool SymbolicDimMgr::Save() { }; // TODO(zhangbopd): update attributes attached in DenseTensorType - for (auto op : *(m_.block())) { - if (!op->HasAttribute(SymbolicDimOp::GetSymbolicDimAttrName())) continue; + for (auto& op : *(m_.block())) { + if (!op.HasAttribute(SymbolicDimOp::GetSymbolicDimAttrName())) continue; auto attrs = - op->attribute(SymbolicDimOp::GetSymbolicDimAttrName()); + op.attribute(SymbolicDimOp::GetSymbolicDimAttrName()); auto symbolic_shape_attr = update_attrs(attrs, [&](const std::string& name) { return symbol_table_.Lookup(name); }); - op->set_attribute(SymbolicDimOp::GetSymbolicDimAttrName(), - symbolic_shape_attr); + op.set_attribute(SymbolicDimOp::GetSymbolicDimAttrName(), + symbolic_shape_attr); } if (!UpdateProductEqualityMap()) { return false; @@ -499,10 +499,10 @@ bool SymbolicDimMgr::Save() { used_symbol_names.push_back(sym.GetSymName()); } }; - for (auto op : *(m_.block())) { - if (!op->HasAttribute(SymbolicDimOp::GetSymbolicDimAttrName())) continue; + for (auto& op : *(m_.block())) { + if (!op.HasAttribute(SymbolicDimOp::GetSymbolicDimAttrName())) continue; auto attrs = - op->attribute(SymbolicDimOp::GetSymbolicDimAttrName()); + op.attribute(SymbolicDimOp::GetSymbolicDimAttrName()); collect_used_symbols(attrs); } auto func_op = symbol_table_.getOp()->dyn_cast(); @@ -559,14 +559,14 @@ bool SymbolicDimMgr::Save() { name_to_symbol[name] = op; } - for (auto op : *(m_.block())) { - if (!op->HasAttribute(SymbolicDimOp::GetSymbolicDimAttrName())) continue; + for (auto& op : *(m_.block())) { + if (!op.HasAttribute(SymbolicDimOp::GetSymbolicDimAttrName())) continue; auto attrs = - op->attribute(SymbolicDimOp::GetSymbolicDimAttrName()); + op.attribute(SymbolicDimOp::GetSymbolicDimAttrName()); auto symbolic_shape_attr = update_attrs( attrs, [&](const std::string& name) { return name_to_symbol[name]; }); - op->set_attribute(SymbolicDimOp::GetSymbolicDimAttrName(), - symbolic_shape_attr); + op.set_attribute(SymbolicDimOp::GetSymbolicDimAttrName(), + symbolic_shape_attr); } // TODO(zhangbopd): update attributes attached to values. @@ -579,11 +579,11 @@ bool SymbolicDimMgr::SaveShapeConstraintGraph() { IR_ENFORCE(func_op); auto op_it = func_op.block()->rbegin(); while (op_it != func_op.block()->rend()) { - if (((*op_it)->isa()) || - ((*op_it)->isa())) + if ((op_it->isa()) || + (op_it->isa())) op_it++; else - op_it = decltype(op_it)(func_op.block()->erase(*(*op_it))); + op_it = decltype(op_it)(func_op.block()->erase(*op_it)); } // save product equal predicate diff --git a/paddle/pir/dialect/shape/utils/shape_utils.cc b/paddle/pir/dialect/shape/utils/shape_utils.cc index 3cee9f5c27671a..1f04e4438e7b69 100644 --- a/paddle/pir/dialect/shape/utils/shape_utils.cc +++ b/paddle/pir/dialect/shape/utils/shape_utils.cc @@ -49,8 +49,8 @@ bool ShapeAnalysis::IsProductEqual( ShapeConstraintIRAnalysis::ShapeConstraintIRAnalysis(ModuleOp m) : m_(m), mgr_(m) { mgr_.Load(); - for (auto op : *(m_.block())) { - auto tie_shape_op = op->dyn_cast(); + for (auto& op : *(m_.block())) { + auto tie_shape_op = op.dyn_cast(); if (!tie_shape_op) continue; Value result = tie_shape_op.input(); auto& symbols = value_to_sym_dims_[result]; diff --git a/paddle/pir/pass/pass.cc b/paddle/pir/pass/pass.cc index d0e3f5d3927a72..1d581e392c3db0 100644 --- a/paddle/pir/pass/pass.cc +++ b/paddle/pir/pass/pass.cc @@ -46,10 +46,10 @@ void detail::PassAdaptor::RunImpl(Operation* op, for (size_t i = 0; i < op->num_regions(); ++i) { auto& region = op->region(i); - for (auto* block : region) { - for (auto op : *block) { - AnalysisManagerHolder am(op, last_am.GetPassInstrumentor()); - if (!RunPipeline(*pm_, op, am, opt_level, verify)) + for (auto& block : region) { + for (auto& op : block) { + AnalysisManagerHolder am(&op, last_am.GetPassInstrumentor()); + if (!RunPipeline(*pm_, &op, am, opt_level, verify)) return SignalPassFailure(); } } diff --git a/paddle/pir/pattern_rewrite/pattern_rewrite_driver.cc b/paddle/pir/pattern_rewrite/pattern_rewrite_driver.cc index ff75f86d6da55a..3c1dcae4bc0e43 100644 --- a/paddle/pir/pattern_rewrite/pattern_rewrite_driver.cc +++ b/paddle/pir/pattern_rewrite/pattern_rewrite_driver.cc @@ -47,8 +47,8 @@ class GreedyPatternRewriteDriver : public pir::PatternRewriter { matcher_.ApplyDefaultCostModel(); if (config.strict_mode != pir::GreedyRewriteStrictness::AnyOp) { for (auto& block : region_) { - for (auto& op_item : *block) { - strict_mode_filtered_ops_.insert(op_item); + for (auto& op_item : block) { + strict_mode_filtered_ops_.insert(&op_item); } } } @@ -67,8 +67,8 @@ class GreedyPatternRewriteDriver : public pir::PatternRewriter { worklist_map_.clear(); for (auto& block_item : region_) { - for (auto& op_item : *block_item) { - worklist_.push_back(op_item); + for (auto& op_item : block_item) { + worklist_.push_back(&op_item); } } if (config_.use_top_down_traversal) { @@ -138,8 +138,8 @@ class GreedyPatternRewriteDriver : public pir::PatternRewriter { for (uint32_t i = 0; i < op->num_regions(); ++i) { auto& region = op->region(i); for (auto& block : region) { - for (auto& op_item : *block) { - RemoveFromWorklist(op_item); + for (auto& op_item : block) { + RemoveFromWorklist(&op_item); } } } diff --git a/test/cpp/pir/cinn/add_broadcast_to_elementwise_test.cc b/test/cpp/pir/cinn/add_broadcast_to_elementwise_test.cc index 801d5644930e1d..7a155fd1d6791b 100644 --- a/test/cpp/pir/cinn/add_broadcast_to_elementwise_test.cc +++ b/test/cpp/pir/cinn/add_broadcast_to_elementwise_test.cc @@ -94,13 +94,13 @@ TEST(PatternRewrite, broadcast_elementwise) { auto it = program.block()->begin(); - CHECK_EQ((*it)->isa(), true); + CHECK_EQ(it->isa(), true); it++; - CHECK_EQ((*it)->isa(), true); + CHECK_EQ(it->isa(), true); it++; - CHECK_EQ((*it)->isa(), true); + CHECK_EQ(it->isa(), true); it++; - CHECK_EQ((*it)->isa(), true); + CHECK_EQ(it->isa(), true); } TEST(PatternRewrite, broadcast_elementwise_both) { @@ -120,15 +120,15 @@ TEST(PatternRewrite, broadcast_elementwise_both) { auto it = program.block()->begin(); - CHECK_EQ((*it)->isa(), true); + CHECK_EQ(it->isa(), true); it++; - CHECK_EQ((*it)->isa(), true); + CHECK_EQ(it->isa(), true); it++; - CHECK_EQ((*it)->isa(), true); + CHECK_EQ(it->isa(), true); it++; - CHECK_EQ((*it)->isa(), true); + CHECK_EQ(it->isa(), true); it++; - CHECK_EQ((*it)->isa(), true); + CHECK_EQ(it->isa(), true); } TEST(PatternRewrite, broadcast_elementwise_sub_both) { @@ -148,13 +148,13 @@ TEST(PatternRewrite, broadcast_elementwise_sub_both) { auto it = program.block()->begin(); - CHECK_EQ((*it)->isa(), true); + CHECK_EQ(it->isa(), true); it++; - CHECK_EQ((*it)->isa(), true); + CHECK_EQ(it->isa(), true); it++; - CHECK_EQ((*it)->isa(), true); + CHECK_EQ(it->isa(), true); it++; - CHECK_EQ((*it)->isa(), true); + CHECK_EQ(it->isa(), true); it++; - CHECK_EQ((*it)->isa(), true); + CHECK_EQ(it->isa(), true); } diff --git a/test/cpp/pir/cinn/build_cinn_pass_test.cc b/test/cpp/pir/cinn/build_cinn_pass_test.cc index e80e88242e0b17..ab874470dab4e9 100644 --- a/test/cpp/pir/cinn/build_cinn_pass_test.cc +++ b/test/cpp/pir/cinn/build_cinn_pass_test.cc @@ -74,8 +74,8 @@ TEST(BuildCinnPassTest, AllOpSupportCinn) { pir::YieldOp::name(), }; int index = 0; - for (auto iter : *group_block) { - CHECK_EQ(iter->name(), op_names[index++]); + for (auto& op : *group_block) { + CHECK_EQ(op.name(), op_names[index++]); } } @@ -121,8 +121,8 @@ TEST(BuildCinnPassTest, NoOpSupportCinn) { paddle::dialect::UnsqueezeOp::name(), }; int index = 0; - for (auto iter : *origin_program->block()) { - CHECK_EQ(iter->name(), op_names[index++]); + for (auto& op : *origin_program->block()) { + CHECK_EQ(op.name(), op_names[index++]); } } @@ -175,8 +175,8 @@ TEST(BuildCinnPassTest, OneCinnSubgraph) { pir::YieldOp::name(), }; int index = 0; - for (auto iter : *group_block) { - CHECK_EQ(iter->name(), op_names[index++]); + for (auto& op : *group_block) { + CHECK_EQ(op.name(), op_names[index++]); } } @@ -230,8 +230,8 @@ TEST(BuildCinnPassTest, MultiCinnSubgraph) { pir::YieldOp::name(), }; int index = 0; - for (auto iter : *group_block) { - CHECK_EQ(iter->name(), op_names_front[index++]); + for (auto& op : *group_block) { + CHECK_EQ(op.name(), op_names_front[index++]); } group_op = origin_program->block()->back(); @@ -243,7 +243,7 @@ TEST(BuildCinnPassTest, MultiCinnSubgraph) { pir::YieldOp::name(), }; index = 0; - for (auto iter : *group_block) { - CHECK_EQ(iter->name(), op_names_back[index++]); + for (auto& op : *group_block) { + CHECK_EQ(op.name(), op_names_back[index++]); } } diff --git a/test/cpp/pir/cinn/dialect_convert_test.cc b/test/cpp/pir/cinn/dialect_convert_test.cc index 91d52f2adc8bdb..398c0892688300 100644 --- a/test/cpp/pir/cinn/dialect_convert_test.cc +++ b/test/cpp/pir/cinn/dialect_convert_test.cc @@ -69,13 +69,13 @@ TEST(DrrTest, reduce_sum) { auto it = program.block()->begin(); - CHECK_EQ((*it)->isa(), true); + CHECK_EQ(it->isa(), true); it++; - CHECK_EQ((*it)->isa(), true); + CHECK_EQ(it->isa(), true); it++; - CHECK_EQ((*it)->isa(), true); + CHECK_EQ(it->isa(), true); it++; - CHECK_EQ((*it)->isa(), true); + CHECK_EQ(it->isa(), true); } TEST(DrrTest, reduce_max) { @@ -91,11 +91,11 @@ TEST(DrrTest, reduce_max) { auto it = program.block()->begin(); - CHECK_EQ((*it)->isa(), true); + CHECK_EQ(it->isa(), true); it++; - CHECK_EQ((*it)->isa(), true); + CHECK_EQ(it->isa(), true); it++; - CHECK_EQ((*it)->isa(), true); + CHECK_EQ(it->isa(), true); it++; - CHECK_EQ((*it)->isa(), true); + CHECK_EQ(it->isa(), true); } diff --git a/test/cpp/pir/cinn/group_op_test.cc b/test/cpp/pir/cinn/group_op_test.cc index 24ebf47a7c84f2..68012ee2b32369 100644 --- a/test/cpp/pir/cinn/group_op_test.cc +++ b/test/cpp/pir/cinn/group_op_test.cc @@ -87,9 +87,9 @@ TEST(GroupOp, TestBuild) { LOG(INFO) << program->block()->size(); std::vector op_num = {2, 5}; int i = 0; - for (auto* sub_op : *(program->block())) { - EXPECT_TRUE(sub_op->isa()); - EXPECT_EQ(sub_op->dyn_cast().ops().size(), + for (auto& sub_op : *(program->block())) { + EXPECT_TRUE(sub_op.isa()); + EXPECT_EQ(sub_op.dyn_cast().ops().size(), op_num[i]); ++i; } @@ -142,9 +142,9 @@ TEST(GroupOp, TestBuildByBlock) { LOG(INFO) << program->block()->size(); std::vector op_num = {2, 5}; int i = 0; - for (auto* sub_op : *(program->block())) { - EXPECT_TRUE(sub_op->isa()); - EXPECT_EQ(sub_op->dyn_cast().ops().size(), + for (auto& sub_op : *(program->block())) { + EXPECT_TRUE(sub_op.isa()); + EXPECT_EQ(sub_op.dyn_cast().ops().size(), op_num[i]); ++i; } diff --git a/test/cpp/pir/cinn/ir_op_fusion_test.cc b/test/cpp/pir/cinn/ir_op_fusion_test.cc index 2b448dcb16fe45..7126bd63f1e682 100644 --- a/test/cpp/pir/cinn/ir_op_fusion_test.cc +++ b/test/cpp/pir/cinn/ir_op_fusion_test.cc @@ -52,9 +52,11 @@ TEST(IROpFusionPass, demo) { auto add = builder.Build(inputs[0], inputs[1]); builder.Build(add.result(0)); - auto res = - cinn::dialect::ir::OpFusionPassInternal(std::vector( - program.block()->begin(), program.block()->end())); + std::vector vec_op; + for (auto& op : *program.block()) { + vec_op.push_back(&op); + } + auto res = cinn::dialect::ir::OpFusionPassInternal(vec_op); ASSERT_EQ(res.size(), 1u); @@ -79,9 +81,12 @@ TEST(IROpFusionPass, ElementWise_Fusion_0) { auto f = builder.Build(e, inputs[2]).result(0); builder.Build(f, inputs[2]); - auto res = - cinn::dialect::ir::OpFusionPassInternal(std::vector( - program.block()->begin(), program.block()->end())); + std::vector vec_op; + for (auto& op : *program.block()) { + vec_op.push_back(&op); + } + + auto res = cinn::dialect::ir::OpFusionPassInternal(vec_op); auto new_group = cinn::dialect::ir::GeneralFusionMergePassInternal(res); @@ -114,9 +119,11 @@ TEST(IROpFusionPass, Broadcast_Test_0) { builder.Build(e, axes, out_shape).result(0); builder.Build(e1, f); - auto res = - cinn::dialect::ir::OpFusionPassInternal(std::vector( - program.block()->begin(), program.block()->end())); + std::vector vec_op; + for (auto& op : *program.block()) { + vec_op.push_back(&op); + } + auto res = cinn::dialect::ir::OpFusionPassInternal(vec_op); auto new_group = cinn::dialect::ir::GeneralFusionMergePassInternal(res); @@ -148,9 +155,11 @@ TEST(IROpFusionPass, Broadcast_Test_1) { builder.Build(e, axes, out_shape).result(0); builder.Build(inputs[3], e1); - auto res = - cinn::dialect::ir::OpFusionPassInternal(std::vector( - program.block()->begin(), program.block()->end())); + std::vector vec_op; + for (auto& op : *program.block()) { + vec_op.push_back(&op); + } + auto res = cinn::dialect::ir::OpFusionPassInternal(vec_op); auto new_group = cinn::dialect::ir::GeneralFusionMergePassInternal(res); @@ -214,9 +223,11 @@ TEST(IROpFusionPass, reduce_test_0) { builder.Build(c, axes, true).result(0); builder.Build(c, axes, true).result(0); - auto res = - cinn::dialect::ir::OpFusionPassInternal(std::vector( - program.block()->begin(), program.block()->end())); + std::vector vec_op; + for (auto& op : *program.block()) { + vec_op.push_back(&op); + } + auto res = cinn::dialect::ir::OpFusionPassInternal(vec_op); auto new_group = cinn::dialect::ir::GeneralFusionMergePassInternal(res); @@ -246,9 +257,11 @@ TEST(IROpFusionPass, reduce_test_1) { builder.Build(c, axes, true).result(0); builder.Build(c, axes1, true).result(0); - auto res = - cinn::dialect::ir::OpFusionPassInternal(std::vector( - program.block()->begin(), program.block()->end())); + std::vector vec_op; + for (auto& op : *program.block()) { + vec_op.push_back(&op); + } + auto res = cinn::dialect::ir::OpFusionPassInternal(vec_op); auto new_group = cinn::dialect::ir::GeneralFusionMergePassInternal(res); @@ -280,9 +293,11 @@ TEST(IROpFusionPass, reduce_test_2) { builder.Build(inputs[2], e).result(0); builder.Build(inputs[2], f).result(0); - auto res = - cinn::dialect::ir::OpFusionPassInternal(std::vector( - program.block()->begin(), program.block()->end())); + std::vector vec_op; + for (auto& op : *program.block()) { + vec_op.push_back(&op); + } + auto res = cinn::dialect::ir::OpFusionPassInternal(vec_op); auto new_group = cinn::dialect::ir::GeneralFusionMergePassInternal(res); @@ -318,9 +333,11 @@ TEST(IROpFusionPass, reduce_test_3) { builder.Build(f, axes1, out_shape).result(0); builder.Build(inputs[2], f1).result(0); - auto res = - cinn::dialect::ir::OpFusionPassInternal(std::vector( - program.block()->begin(), program.block()->end())); + std::vector vec_op; + for (auto& op : *program.block()) { + vec_op.push_back(&op); + } + auto res = cinn::dialect::ir::OpFusionPassInternal(vec_op); auto new_group = cinn::dialect::ir::GeneralFusionMergePassInternal(res); @@ -392,9 +409,11 @@ TEST(IROpFusionPass, reduce_test_5) { builder.Build(inputs[1], axes, false).result(0); builder.Build(c, axes, false).result(0); - auto res = - cinn::dialect::ir::OpFusionPassInternal(std::vector( - program.block()->begin(), program.block()->end())); + std::vector vec_op; + for (auto& op : *program.block()) { + vec_op.push_back(&op); + } + auto res = cinn::dialect::ir::OpFusionPassInternal(vec_op); auto new_group = cinn::dialect::ir::GeneralFusionMergePassInternal(res); @@ -468,9 +487,11 @@ TEST(IROpFusionPass, layer_norm) { auto t5 = builder.Build(t3, scale).result(0); builder.Build(t5, bias).result(0); - auto res = - cinn::dialect::ir::OpFusionPassInternal(std::vector( - program.block()->begin(), program.block()->end())); + std::vector vec_op; + for (auto& op : *program.block()) { + vec_op.push_back(&op); + } + auto res = cinn::dialect::ir::OpFusionPassInternal(vec_op); auto new_group = cinn::dialect::ir::GeneralFusionMergePassInternal(res); @@ -516,9 +537,11 @@ TEST(IROpFusionPass, softmax) { auto divide = builder.Build(exp, broadcast_2).result(0); - auto res = - cinn::dialect::ir::OpFusionPassInternal(std::vector( - program.block()->begin(), program.block()->end())); + std::vector vec_op; + for (auto& op : *program.block()) { + vec_op.push_back(&op); + } + auto res = cinn::dialect::ir::OpFusionPassInternal(vec_op); auto new_group = cinn::dialect::ir::GeneralFusionMergePassInternal(res); @@ -589,9 +612,11 @@ TEST(IROpFusionPass, layer_norm2) { builder.Build(mean2, std::vector({-1})) .result(0); - auto res = - cinn::dialect::ir::OpFusionPassInternal(std::vector( - program.block()->begin(), program.block()->end())); + std::vector vec_op; + for (auto& op : *program.block()) { + vec_op.push_back(&op); + } + auto res = cinn::dialect::ir::OpFusionPassInternal(vec_op); auto new_group = cinn::dialect::ir::GeneralFusionMergePassInternal(res); ASSERT_EQ(new_group.size(), 1u); ASSERT_EQ(new_group[0]->ops.size(), program.block()->size()); diff --git a/test/cpp/pir/cinn/jit_instruction_test.cc b/test/cpp/pir/cinn/jit_instruction_test.cc index 5291ce0582e2f6..4382b7ffa045f6 100644 --- a/test/cpp/pir/cinn/jit_instruction_test.cc +++ b/test/cpp/pir/cinn/jit_instruction_test.cc @@ -99,11 +99,11 @@ TEST(CinnJitInstruction, Run) { std::unordered_map value_map; for (auto it = program->block()->begin(); it != program->block()->end(); ++it) { - if (checking_cinn_ops.count((*it)->name())) { + if (checking_cinn_ops.count(it->name())) { auto ir_compiler = new cinn::hlir::framework::PirCompiler(*program, target, scope); - std::vector<::pir::Operation*> ops = {*it}; + std::vector<::pir::Operation*> ops = {it}; auto group = std::make_shared(ops); auto fn_ptr_res = ir_compiler->BuildCUDAJITInfo({group}); compiler_list.push_back(ir_compiler); @@ -112,35 +112,35 @@ TEST(CinnJitInstruction, Run) { cinn::dialect::CUDAJITInfoAttribute::get(ctx, fn_ptr_res[0])}, }; - auto out_type = (*it)->result(0).type(); + auto out_type = it->result(0).type(); std::vector vec_ins; - for (size_t i = 0; i < (*it)->num_operands(); ++i) { - vec_ins.push_back(value_map.at((*it)->operand_source(i))); + for (size_t i = 0; i < it->num_operands(); ++i) { + vec_ins.push_back(value_map.at(it->operand_source(i))); } ::pir::Operation* cinn_op = ::pir::Operation::Create(vec_ins, op_attrs, {out_type}, op_info); - value_map[(*it)->result(0)] = cinn_op->result(0); + value_map[it->result(0)] = cinn_op->result(0); ir_program->block()->push_back(cinn_op); } else { std::vector vec_ins; - for (size_t i = 0; i < (*it)->num_operands(); ++i) { - vec_ins.push_back(value_map.at((*it)->operand_source(i))); + for (size_t i = 0; i < it->num_operands(); ++i) { + vec_ins.push_back(value_map.at(it->operand_source(i))); } - auto type1 = (*it)->result(0).type(); - ::pir::OpInfo info1 = ctx->GetRegisteredOpInfo((*it)->name()); - ::pir::Operation* op = ::pir::Operation::Create( - vec_ins, (*it)->attributes(), {type1}, info1); + auto type1 = it->result(0).type(); + ::pir::OpInfo info1 = ctx->GetRegisteredOpInfo(it->name()); + ::pir::Operation* op = + ::pir::Operation::Create(vec_ins, it->attributes(), {type1}, info1); ir_program->block()->push_back(op); - value_map[(*it)->result(0)] = op->result(0); + value_map[it->result(0)] = op->result(0); } } diff --git a/test/cpp/pir/core/program_translator_test.cc b/test/cpp/pir/core/program_translator_test.cc index ce94d803a99e87..af03ef0f1651a7 100644 --- a/test/cpp/pir/core/program_translator_test.cc +++ b/test/cpp/pir/core/program_translator_test.cc @@ -89,104 +89,104 @@ TEST(OperatorDialectTest, ConditionBlock) { size_t id = 0; for (auto &op : *program->block()) { if (id == 0 || id == 1) { - EXPECT_EQ(op->isa(), true); + EXPECT_EQ(op.isa(), true); } if (id == 2) { - EXPECT_EQ(op->isa(), true); + EXPECT_EQ(op.isa(), true); } if (id == 3) { - EXPECT_EQ(op->isa(), true); - EXPECT_EQ(op->num_regions(), 2u); + EXPECT_EQ(op.isa(), true); + EXPECT_EQ(op.num_regions(), 2u); // true block pir::Block *true_block = - op->dyn_cast().true_block(); + op.dyn_cast().true_block(); size_t true_id = 0; for (auto &op1 : *true_block) { if (true_id == 0 || true_id == 1) { - EXPECT_EQ(op1->isa(), true); + EXPECT_EQ(op1.isa(), true); } if (true_id == 2) { - EXPECT_EQ(op1->isa(), true); + EXPECT_EQ(op1.isa(), true); } if (true_id == 3) { pir::Block *true_true_block = - op1->dyn_cast().true_block(); + op1.dyn_cast().true_block(); size_t true_true_id = 0; for (auto &op2 : *true_true_block) { if (true_true_id == 0) { - EXPECT_EQ(op2->isa(), true); + EXPECT_EQ(op2.isa(), true); } if (true_true_id == 1) { - EXPECT_EQ(op2->isa(), true); + EXPECT_EQ(op2.isa(), true); } true_true_id++; } pir::Block *false_false_block = - op1->dyn_cast().false_block(); + op1.dyn_cast().false_block(); size_t false_false_id = 0; for (auto &op2 : *false_false_block) { if (false_false_id == 0) { - EXPECT_EQ(op2->isa(), true); + EXPECT_EQ(op2.isa(), true); } if (false_false_id == 1) { - EXPECT_EQ(op2->isa(), true); + EXPECT_EQ(op2.isa(), true); } false_false_id++; } } if (true_id == 4) { - EXPECT_EQ(op1->isa(), true); + EXPECT_EQ(op1.isa(), true); } if (true_id == 5) { - EXPECT_EQ(op1->isa(), true); + EXPECT_EQ(op1.isa(), true); } true_id++; } // false block pir::Block *false_block = - op->dyn_cast().false_block(); + op.dyn_cast().false_block(); size_t false_id = 0; for (auto &op1 : *false_block) { if (false_id == 0 || false_id == 1) { - EXPECT_EQ(op1->isa(), true); + EXPECT_EQ(op1.isa(), true); } if (false_id == 2) { - EXPECT_EQ(op1->isa(), true); + EXPECT_EQ(op1.isa(), true); } if (false_id == 3) { - EXPECT_EQ(op1->isa(), true); + EXPECT_EQ(op1.isa(), true); // true block pir::Block *false_true_block = - op1->dyn_cast().true_block(); + op1.dyn_cast().true_block(); size_t false_true_id = 0; for (auto &op2 : *false_true_block) { if (false_true_id == 0) { - EXPECT_EQ(op2->isa(), true); + EXPECT_EQ(op2.isa(), true); } if (false_true_id == 1) { - EXPECT_EQ(op2->isa(), true); + EXPECT_EQ(op2.isa(), true); } false_true_id++; } // false block pir::Block *false_false_block = - op1->dyn_cast().true_block(); + op1.dyn_cast().true_block(); size_t false_false_id = 0; for (auto &op2 : *false_false_block) { if (false_false_id == 0) { - EXPECT_EQ(op2->isa(), true); + EXPECT_EQ(op2.isa(), true); } if (false_false_id == 1) { - EXPECT_EQ(op2->isa(), true); + EXPECT_EQ(op2.isa(), true); } false_false_id++; } } if (false_id == 4) { - EXPECT_EQ(op1->isa(), true); + EXPECT_EQ(op1.isa(), true); } if (false_id == 5) { - EXPECT_EQ(op1->isa(), true); + EXPECT_EQ(op1.isa(), true); } false_id++; } @@ -284,59 +284,59 @@ TEST(OperatorDialectTest, WhileOpProgram) { size_t id = 0; for (auto &op : *program->block()) { if (id == 0 || id == 1) { - EXPECT_TRUE(op->isa()); + EXPECT_TRUE(op.isa()); } if (id == 2) { - EXPECT_TRUE(op->isa()); + EXPECT_TRUE(op.isa()); } if (id == 3) { - EXPECT_TRUE(op->isa()); - EXPECT_EQ(op->num_regions(), 1u); + EXPECT_TRUE(op.isa()); + EXPECT_EQ(op.num_regions(), 1u); // body block pir::Block *body_block = - op->dyn_cast().body_block(); + op.dyn_cast().body_block(); size_t body_id = 0; for (auto &op1 : *body_block) { if (body_id == 0) { - EXPECT_TRUE(op1->isa()); + EXPECT_TRUE(op1.isa()); } if (body_id == 1) { - EXPECT_TRUE(op1->isa()); + EXPECT_TRUE(op1.isa()); } if (body_id == 2) { - EXPECT_TRUE(op1->isa()); + EXPECT_TRUE(op1.isa()); } if (body_id == 3) { pir::Block *body_body_block = - op1->dyn_cast().body_block(); + op1.dyn_cast().body_block(); size_t body_body_id = 0; for (auto &op2 : *body_body_block) { if (body_body_id == 0) { - EXPECT_TRUE(op2->isa()); + EXPECT_TRUE(op2.isa()); } if (body_body_id == 1) { - EXPECT_TRUE(op2->isa()); + EXPECT_TRUE(op2.isa()); } if (body_body_id == 2) { - EXPECT_TRUE(op2->isa()); + EXPECT_TRUE(op2.isa()); } if (body_body_id == 3 || body_body_id == 4) { - EXPECT_TRUE(op2->isa()); + EXPECT_TRUE(op2.isa()); } if (body_body_id == 5) { - EXPECT_TRUE(op2->isa()); + EXPECT_TRUE(op2.isa()); } body_body_id++; } } if (body_id == 4) { - EXPECT_TRUE(op1->isa()); + EXPECT_TRUE(op1.isa()); } if (body_id == 5 || body_id == 6) { - EXPECT_TRUE(op1->isa()); + EXPECT_TRUE(op1.isa()); } if (body_id == 7) { - EXPECT_TRUE(op1->isa()); + EXPECT_TRUE(op1.isa()); } body_id++; } diff --git a/test/cpp/pir/pass/pass_manager_test.cc b/test/cpp/pir/pass/pass_manager_test.cc index 03e7d88d484bca..7c00a5d24cb988 100644 --- a/test/cpp/pir/pass/pass_manager_test.cc +++ b/test/cpp/pir/pass/pass_manager_test.cc @@ -104,10 +104,8 @@ struct CountOpAnalysis { LOG(INFO) << "In CountOpAnalysis, op is " << container_op->name() << "\n"; for (size_t i = 0; i < container_op->num_regions(); ++i) { auto ®ion = container_op->region(i); - for (auto block : region) { - for (auto it = block->begin(); it != block->end(); ++it) { - ++count; - } + for (auto &block : region) { + count += block.size(); } } diff --git a/test/cpp/pir/pattern_rewrite/drr_fuse_linear_param_grad_add_test.cc b/test/cpp/pir/pattern_rewrite/drr_fuse_linear_param_grad_add_test.cc index da496e6d940c0e..b55fbf1e0801f3 100644 --- a/test/cpp/pir/pattern_rewrite/drr_fuse_linear_param_grad_add_test.cc +++ b/test/cpp/pir/pattern_rewrite/drr_fuse_linear_param_grad_add_test.cc @@ -155,8 +155,8 @@ void BuildProgram3(pir::Builder &builder) { // NOLINT } bool verify_pass(const pir::Program &program) { - for (auto op : *(program.block())) { - if (op->name() == paddle::dialect::FusedLinearParamGradAddOp::name()) { + for (auto &op : *(program.block())) { + if (op.name() == paddle::dialect::FusedLinearParamGradAddOp::name()) { return true; } } diff --git a/test/cpp/prim/test_vjp.cc b/test/cpp/prim/test_vjp.cc index 81e3bfde17a62b..2c3ba7073e6065 100644 --- a/test/cpp/prim/test_vjp.cc +++ b/test/cpp/prim/test_vjp.cc @@ -44,9 +44,9 @@ namespace framework { pir::Operation* GetOpFromProgram(const std::string& op_name, const pir::Program& program) { - for (auto op : *(program.block())) { - if (op->name() == op_name) { - return op; + for (auto& op : *(program.block())) { + if (op.name() == op_name) { + return &op; } } return nullptr; diff --git a/test/ir/pir/test_if_api.py b/test/ir/pir/test_if_api.py index 9c55aa0a5f4943..e69967b4dd8bb8 100644 --- a/test/ir/pir/test_if_api.py +++ b/test/ir/pir/test_if_api.py @@ -15,6 +15,7 @@ import unittest import paddle +from paddle.base.libpaddle.pir import get_used_external_value paddle.enable_static() @@ -39,11 +40,11 @@ def test_if_with_single_output(self): x = paddle.static.data(name="x", shape=[6, 1], dtype="float32") y = paddle.static.data(name="y", shape=[6, 1], dtype="float32") out = paddle.static.nn.cond(x < y, lambda: x + y, lambda: x - y) - self.assertEqual( - out[0].get_defining_op().name(), - "pd_op.if", - ) + if_op = out[0].get_defining_op() + self.assertEqual(if_op.name(), "pd_op.if") self.assertEqual(len(out), 1) + value_list = get_used_external_value(if_op) + print(value_list) def test_if_with_multiple_output(self): main_program = paddle.static.Program() From 1585549aa28e32c160004fd20352311ceb7d5a23 Mon Sep 17 00:00:00 2001 From: Ryan <44900829+DrRyanHuang@users.noreply.github.com> Date: Mon, 20 Nov 2023 14:39:36 +0800 Subject: [PATCH 17/46] =?UTF-8?q?=E3=80=90PIR=20API=20adaptor=20No.140-142?= =?UTF-8?q?=E3=80=91=20Migrate=20lstsq/lu/lu=5Funpack=20into=20pir=20(#588?= =?UTF-8?q?15)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- python/paddle/device/__init__.py | 2 +- python/paddle/tensor/linalg.py | 35 +++++++++++++++++++----- test/legacy_test/test_linalg_lstsq_op.py | 10 +++++-- test/legacy_test/test_lu_op.py | 11 +++++--- test/legacy_test/test_lu_unpack_op.py | 14 ++++++---- 5 files changed, 52 insertions(+), 20 deletions(-) diff --git a/python/paddle/device/__init__.py b/python/paddle/device/__init__.py index f6c3bfc78a9a64..e5679f0efc7700 100644 --- a/python/paddle/device/__init__.py +++ b/python/paddle/device/__init__.py @@ -308,7 +308,7 @@ def get_device(): """ device = '' - place = framework._current_expected_place() + place = framework._current_expected_place_() if isinstance(place, core.CPUPlace): device = 'cpu' elif isinstance(place, core.CUDAPlace): diff --git a/python/paddle/tensor/linalg.py b/python/paddle/tensor/linalg.py index d4aba965daf51d..5088cea790fd2c 100644 --- a/python/paddle/tensor/linalg.py +++ b/python/paddle/tensor/linalg.py @@ -2451,7 +2451,7 @@ def lu(x, pivot=True, get_infos=False, name=None): >>> # one can verify : X = P @ L @ U ; """ - if in_dynamic_mode(): + if in_dynamic_or_pir_mode(): lu, p, info = _C_ops.lu(x, pivot) else: check_variable_and_dtype(x, 'dtype', ['float32', 'float64'], 'lu') @@ -2554,7 +2554,7 @@ def lu_unpack(x, y, unpack_ludata=True, unpack_pivots=True, name=None): raise ValueError( f"The shape of Pivots should be (*, K), but received ndim is [{y.ndim} < 1]" ) - if in_dynamic_mode(): + if in_dynamic_or_pir_mode(): P, L, U = _C_ops.lu_unpack(x, y, unpack_ludata, unpack_pivots) return P, L, U else: @@ -3454,7 +3454,16 @@ def lstsq(x, y, rcond=None, driver=None, name=None): else: raise RuntimeError("Only support lstsq api for CPU or CUDA device.") - if not (x.dtype == y.dtype and x.dtype in (paddle.float32, paddle.float64)): + if not ( + x.dtype == y.dtype + and x.dtype + in ( + paddle.float32, + paddle.float64, + paddle.base.core.DataType.FLOAT32, + paddle.base.core.DataType.FLOAT64, + ) + ): raise ValueError( "Only support x and y have the same dtype such as 'float32' and 'float64'." ) @@ -3475,17 +3484,29 @@ def lstsq(x, y, rcond=None, driver=None, name=None): ) if rcond is None: - if x.dtype == paddle.float32: + if ( + x.dtype == paddle.float32 + or x.dtype == paddle.base.core.DataType.FLOAT32 + ): rcond = 1e-7 * max(x.shape[-2], x.shape[-1]) - elif x.dtype == paddle.float64: + elif ( + x.dtype == paddle.float64 + or x.dtype == paddle.base.core.DataType.FLOAT64 + ): rcond = 1e-15 * max(x.shape[-2], x.shape[-1]) - if in_dynamic_mode(): + if in_dynamic_or_pir_mode(): solution, residuals, rank, singular_values = _C_ops.lstsq( x, y, rcond, driver ) if driver == "gels": - rank = paddle.empty(shape=[0], dtype=paddle.int32) + if in_dynamic_mode(): + rank = paddle.empty(shape=[0], dtype=paddle.int32) + + else: + rank = paddle.empty( + shape=[0], dtype=paddle.base.core.DataType.INT32 + ) singular_values = paddle.empty(shape=[0], dtype=x.dtype) elif driver == "gelsy": singular_values = paddle.empty(shape=[0], dtype=x.dtype) diff --git a/test/legacy_test/test_linalg_lstsq_op.py b/test/legacy_test/test_linalg_lstsq_op.py index ca9ec7dfb26d51..c54724a16f6fbb 100644 --- a/test/legacy_test/test_linalg_lstsq_op.py +++ b/test/legacy_test/test_linalg_lstsq_op.py @@ -19,6 +19,7 @@ import paddle from paddle import base from paddle.base import core +from paddle.pir_utils import test_with_pir_api class LinalgLstsqTestCase(unittest.TestCase): @@ -26,7 +27,7 @@ def setUp(self): self.devices = ["cpu"] self.init_config() if core.is_compiled_with_cuda() and self.driver == "gels": - self.devices.append("gpu:0") + self.devices.append("gpu") self.generate_input() self.generate_output() np.random.seed(2022) @@ -91,12 +92,15 @@ def test_eager_dygraph(self): self._result_sg_values = results[3].numpy() self.assert_np_close() + @test_with_pir_api def test_static(self): paddle.enable_static() for dev in self.devices: paddle.set_device(dev) place = base.CPUPlace() if dev == "cpu" else base.CUDAPlace(0) - with base.program_guard(base.Program(), base.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): x = paddle.static.data( name="x", shape=self._input_shape_1, @@ -112,7 +116,6 @@ def test_static(self): ) exe = base.Executor(place) fetches = exe.run( - base.default_main_program(), feed={"x": self._input_data_1, "y": self._input_data_2}, fetch_list=[results], ) @@ -282,6 +285,7 @@ class TestLinalgLstsqAPIError(unittest.TestCase): def setUp(self): pass + @test_with_pir_api def test_api_errors(self): def test_x_bad_shape(): x = paddle.to_tensor(np.random.random(size=(5)), dtype=np.float32) diff --git a/test/legacy_test/test_lu_op.py b/test/legacy_test/test_lu_op.py index b875be084fffc2..a140afb1e675f1 100644 --- a/test/legacy_test/test_lu_op.py +++ b/test/legacy_test/test_lu_op.py @@ -24,6 +24,7 @@ import paddle from paddle import base from paddle.base import core +from paddle.pir_utils import test_with_pir_api def scipy_lu(A, pivot): @@ -156,10 +157,10 @@ def setUp(self): } def test_check_output(self): - self.check_output() + self.check_output(check_pir=True) def test_check_grad(self): - self.check_grad(['X'], ['Out']) + self.check_grad(['X'], ['Out'], check_pir=True) # m = n 2D @@ -238,6 +239,7 @@ def run_lu_dygraph(shape, dtype): for tensor_shape, dtype in itertools.product(tensor_shapes, dtypes): run_lu_dygraph(tensor_shape, dtype) + @test_with_pir_api def test_static(self): paddle.enable_static() @@ -257,7 +259,9 @@ def run_lu_static(shape, dtype): if core.is_compiled_with_cuda(): places.append(base.CUDAPlace(0)) for place in places: - with base.program_guard(base.Program(), base.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): batch_size = a.size // (a.shape[-1] * a.shape[-2]) sP, sl, sU = scipy_lu(a, pivot) sL = np.tril(sl, -1) @@ -284,7 +288,6 @@ def run_lu_static(shape, dtype): lu, p = paddle.linalg.lu(x, pivot=pivot) exe = base.Executor(place) fetches = exe.run( - base.default_main_program(), feed={"input": a}, fetch_list=[lu, p], ) diff --git a/test/legacy_test/test_lu_unpack_op.py b/test/legacy_test/test_lu_unpack_op.py index 57cabc4872d98f..91888b26d86604 100644 --- a/test/legacy_test/test_lu_unpack_op.py +++ b/test/legacy_test/test_lu_unpack_op.py @@ -24,6 +24,7 @@ import paddle from paddle import base from paddle.base import core +from paddle.pir_utils import test_with_pir_api def scipy_lu_unpack(A): @@ -138,7 +139,9 @@ def setUp(self): lu = lu.numpy() pivots = pivots.numpy() else: - with base.program_guard(base.Program(), base.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): place = base.CPUPlace() if core.is_compiled_with_cuda(): place = base.CUDAPlace(0) @@ -148,7 +151,6 @@ def setUp(self): lu, p = paddle.linalg.lu(xv) exe = base.Executor(place) fetches = exe.run( - base.default_main_program(), feed={"input": x}, fetch_list=[lu, p], ) @@ -168,7 +170,7 @@ def setUp(self): } def test_check_output(self): - self.check_output() + self.check_output(check_pir=True) def test_check_grad(self): self.check_grad(['X'], ['L', 'U']) @@ -258,6 +260,7 @@ def run_lu_unpack_dygraph(shape, dtype): for tensor_shape, dtype in itertools.product(tensor_shapes, dtypes): run_lu_unpack_dygraph(tensor_shape, dtype) + @test_with_pir_api def test_static(self): paddle.enable_static() @@ -275,7 +278,9 @@ def run_lu_static(shape, dtype): if core.is_compiled_with_cuda(): places.append(base.CUDAPlace(0)) for place in places: - with base.program_guard(base.Program(), base.Program()): + with paddle.static.program_guard( + paddle.static.Program(), paddle.static.Program() + ): sP, sL, sU = scipy_lu_unpack(a) x = paddle.static.data( @@ -285,7 +290,6 @@ def run_lu_static(shape, dtype): pP, pL, pU = paddle.linalg.lu_unpack(lu, p) exe = base.Executor(place) fetches = exe.run( - base.default_main_program(), feed={"input": a}, fetch_list=[pP, pL, pU], ) From 9608fc3e3007fcbbfad0bca5c7aeca9c200e3e08 Mon Sep 17 00:00:00 2001 From: wanghuancoder Date: Mon, 20 Nov 2023 14:41:01 +0800 Subject: [PATCH 18/46] convert disttensor for run program (#59143) --- .../fluid/pybind/eager_legacy_custom_python_api.h | 14 ++++++++++++++ paddle/fluid/pybind/eager_utils.cc | 5 +++-- paddle/fluid/pybind/eager_utils.h | 3 ++- 3 files changed, 19 insertions(+), 3 deletions(-) diff --git a/paddle/fluid/pybind/eager_legacy_custom_python_api.h b/paddle/fluid/pybind/eager_legacy_custom_python_api.h index 682995f9874fc3..601bf99aebeb60 100644 --- a/paddle/fluid/pybind/eager_legacy_custom_python_api.h +++ b/paddle/fluid/pybind/eager_legacy_custom_python_api.h @@ -31,6 +31,13 @@ static PyObject *eager_api_run_program(PyObject *self, // TOREMOVE auto Out = GetTensorPtrListFromArgs("run_program", "Out", args, 2, true); auto OutScope = GetScopePtrListFromArgs("run_program", "OutScope", args, 3, false); + const phi::distributed::ProcessMesh *mesh = nullptr; + if (InputsContainDistTensor(&mesh, X, Params, Out)) { + X = GetTensorListFromArgs("run_program", "X", args, 0, true, mesh); + Params = + GetTensorListFromArgs("run_program", "Params", args, 1, true, mesh); + Out = GetTensorPtrListFromArgs("run_program", "Out", args, 2, true, mesh); + } framework::AttributeMap attrs; // TODO(zengjinle): support CUDA Graph on eager mode ConstructAttrMapFromPyArgs( @@ -70,6 +77,13 @@ static PyObject *pir_eager_api_run_program(PyObject *self, auto Out = GetTensorPtrListFromArgs("run_program", "Out", args, 2, true); auto OutScope = GetScopePtrListFromArgs("run_program", "OutScope", args, 3, false); + const phi::distributed::ProcessMesh *mesh = nullptr; + if (InputsContainDistTensor(&mesh, X, Params, Out)) { + X = GetTensorListFromArgs("run_program", "X", args, 0, true, mesh); + Params = + GetTensorListFromArgs("run_program", "Params", args, 1, true, mesh); + Out = GetTensorPtrListFromArgs("run_program", "Out", args, 2, true, mesh); + } framework::AttributeMap attrs; // TODO(zengjinle): support CUDA Graph on eager mode VLOG(1) << "Start Pir ConstructAttrMapFromPyArgs"; diff --git a/paddle/fluid/pybind/eager_utils.cc b/paddle/fluid/pybind/eager_utils.cc index 9cb81e16cfba4a..4c164863754cfd 100644 --- a/paddle/fluid/pybind/eager_utils.cc +++ b/paddle/fluid/pybind/eager_utils.cc @@ -1586,7 +1586,8 @@ std::vector GetTensorPtrListFromArgs( const std::string& arg_name, PyObject* args, ssize_t arg_idx, - bool dispensable) { + bool dispensable, + const phi::distributed::ProcessMesh* mesh) { PyObject* list = PyTuple_GET_ITEM(args, arg_idx); if (list == nullptr) { @@ -1602,7 +1603,7 @@ std::vector GetTensorPtrListFromArgs( } std::vector result; - const phi::distributed::ProcessMesh* local_mesh = nullptr; + const phi::distributed::ProcessMesh* local_mesh = mesh; int mesh_start_index = -1; if (PyList_Check(list)) { diff --git a/paddle/fluid/pybind/eager_utils.h b/paddle/fluid/pybind/eager_utils.h index 7a25bc28ba3a4b..0d53735574180d 100644 --- a/paddle/fluid/pybind/eager_utils.h +++ b/paddle/fluid/pybind/eager_utils.h @@ -370,7 +370,8 @@ std::vector GetTensorPtrListFromArgs( const std::string& arg_name, PyObject* args, ssize_t arg_idx, - bool dispensable = false); + bool dispensable = false, + const phi::distributed::ProcessMesh* mesh = nullptr); std::vector GetTensorPtrListFromPyObject(PyObject* obj); From 6e6f9ef6a89d391828b874bd9af51a6c50bf334c Mon Sep 17 00:00:00 2001 From: Zhiheng Liu <133497712+zhiheng-liu@users.noreply.github.com> Date: Mon, 20 Nov 2023 14:42:12 +0800 Subject: [PATCH 19/46] Fixed clang-tidy check failed under windows issue. (#59134) --- tools/codestyle/clang-tidy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/codestyle/clang-tidy.py b/tools/codestyle/clang-tidy.py index d8f87d1a630d70..404413b9b99457 100644 --- a/tools/codestyle/clang-tidy.py +++ b/tools/codestyle/clang-tidy.py @@ -471,7 +471,7 @@ def main(): if __name__ == '__main__': target_version = "15.0.2" try: - out = subprocess.check_output(['clang-tidy --version'], shell=True) + out = subprocess.check_output(['clang-tidy', '--version'], shell=True) version = out.decode('utf-8') if version.find(target_version) == -1: print( From 42a5aed85d766fe20116f39bcb55189f22a088b4 Mon Sep 17 00:00:00 2001 From: kangguangli Date: Mon, 20 Nov 2023 14:48:09 +0800 Subject: [PATCH 20/46] [PIR] set lr scheduler in pir (#58973) * set lr scheduler in pir * add unittest * fix * fix --- python/paddle/base/executor.py | 28 +++++ python/paddle/optimizer/optimizer.py | 84 ++++++++++---- test/ir/pir/CMakeLists.txt | 1 + test/ir/pir/test_lr_scheduler_in_pir.py | 146 ++++++++++++++++++++++++ 4 files changed, 236 insertions(+), 23 deletions(-) create mode 100644 test/ir/pir/test_lr_scheduler_in_pir.py diff --git a/python/paddle/base/executor.py b/python/paddle/base/executor.py index c829fdfa17819d..ee586155360a16 100755 --- a/python/paddle/base/executor.py +++ b/python/paddle/base/executor.py @@ -2019,6 +2019,34 @@ def _run_pir_impl( self._pir_feed_data(program, feed, scope) + if hasattr(program, 'lr_scheduler'): + from paddle.optimizer.lr import LRScheduler + + assert isinstance( + program.lr_scheduler, LRScheduler + ), "must be LRScheduler" + + lr_scheduler = program.lr_scheduler + lr_value = lr_scheduler() + lr_var = program.lr_var + + data = np.array([lr_value]).astype(convert_dtype(lr_var.dtype)) + tensor = core.get_variable_tensor( + global_scope(), lr_scheduler._var_name + ) + # NOTE(dev): `tensor.set(data, self.place)` always call TensorCopySync that is a blocking behavior. So we use `_copy_from` to replace it. + cpu_tensor = _as_lodtensor(data, core.CPUPlace()) + if core.is_cuda_graph_capturing(): + warnings.warn( + "Caution!!! When capturing CUDA Graph, the learning rate scheduler would not " + "take any effect! Please set the learning rate manually before each batch!" + ) + elif core.is_compiled_with_ipu(): + # for ipu, tensor is allocated on cpu + tensor._copy_from(cpu_tensor, tensor._place()) + else: + tensor._copy_from(cpu_tensor, self.place) + ret = new_exe.run(list(feed.keys()), return_numpy) return ret diff --git a/python/paddle/optimizer/optimizer.py b/python/paddle/optimizer/optimizer.py index 5bd8ea801f36be..0754494ef58455 100644 --- a/python/paddle/optimizer/optimizer.py +++ b/python/paddle/optimizer/optimizer.py @@ -20,6 +20,7 @@ import paddle import paddle.autograd as imperative_base from paddle import _C_ops +from paddle._pir_ops import get_parameter, set_parameter from paddle.base import core from paddle.base.framework import ( Variable, @@ -453,29 +454,63 @@ def do_create(): if isinstance(self._learning_rate, LRScheduler): lr_var = self._global_learning_rate() # only create global lr_var once - if not isinstance(lr_var, framework.Variable): + if in_pir_mode(): + startup_program = paddle.static.default_startup_program() + main_program = paddle.static.default_main_program() + lr_name = unique_name.generate('learning_rate') - self._learning_rate._var_name = lr_name - lr_var = self.helper.create_global_variable( - name=lr_name, - shape=[], - persistable=True, - stop_gradient=True, - dtype=_lr_dtype, + # startup program insert && set_parameter + lr_value = float(self._learning_rate()) + with paddle.static.program_guard(startup_program): + initializer = paddle.nn.initializer.Constant( + value=lr_value + ) + paramete_meta = paddle.pir.core.ParameterMeta( + [], _lr_dtype + ) + init_result = initializer( + paramete_meta, startup_program.global_block() + ) + init_result.persistable = True + set_parameter(init_result, lr_name) + main_program.move_parameters_from(startup_program) + + if not isinstance(lr_var, paddle.pir.OpResult): + self._learning_rate._var_name = lr_name + with paddle.static.program_guard(main_program): + param = get_parameter(lr_name, _lr_dtype, []) + param.stop_gradient = True + param.persistable = True + main_program.lr_scheduler = self._learning_rate + main_program.lr_var = param + self._learning_rate_map[main_program] = param + + else: + if not isinstance(lr_var, framework.Variable): + lr_name = unique_name.generate('learning_rate') + self._learning_rate._var_name = lr_name + lr_var = self.helper.create_global_variable( + name=lr_name, + shape=[], + persistable=True, + stop_gradient=True, + dtype=_lr_dtype, + ) + main_prog = framework.default_main_program() + main_prog.lr_scheduler = self._learning_rate + main_prog.lr_var = lr_var + + self._learning_rate_map[ + framework.default_main_program() + ] = lr_var + + lr_value = float(self._learning_rate()) + self.helper.set_variable_initializer( + lr_var, + initializer=paddle.nn.initializer.Constant( + value=lr_value + ), ) - main_prog = framework.default_main_program() - main_prog.lr_scheduler = self._learning_rate - main_prog.lr_var = lr_var - - self._learning_rate_map[ - framework.default_main_program() - ] = lr_var - - lr_value = float(self._learning_rate()) - self.helper.set_variable_initializer( - lr_var, - initializer=paddle.nn.initializer.Constant(value=lr_value), - ) elif isinstance(self._learning_rate, float): # only create global lr_var once lr = self._global_learning_rate() @@ -491,7 +526,7 @@ def do_create(): ) ) self._learning_rate_map[ - framework.default_main_program() + paddle.static.default_main_program() ] = paddle._pir_ops.full( [], self._learning_rate, @@ -727,7 +762,10 @@ def _global_learning_rate(self, program=None): :return: """ if program is None: - program = framework.default_main_program() + if in_dygraph_mode(): + program = framework.default_main_program() + else: + program = paddle.static.default_main_program() return self._learning_rate_map.get(program, None) def _append_optimize_op(self, block, param_and_grad): diff --git a/test/ir/pir/CMakeLists.txt b/test/ir/pir/CMakeLists.txt index df846ad2caec14..5a9f2c48509b3f 100644 --- a/test/ir/pir/CMakeLists.txt +++ b/test/ir/pir/CMakeLists.txt @@ -7,6 +7,7 @@ string(REPLACE ".py" "" TEST_INTERP_CASES "${TEST_INTERP_CASES}") set(TEST_IR_SYSTEM_CASES test_build_model test_if_api + test_lr_scheduler_in_pir test_pd_inplace_pass test_symbol_overload test_pir_to_static diff --git a/test/ir/pir/test_lr_scheduler_in_pir.py b/test/ir/pir/test_lr_scheduler_in_pir.py new file mode 100644 index 00000000000000..174086e65ed2cc --- /dev/null +++ b/test/ir/pir/test_lr_scheduler_in_pir.py @@ -0,0 +1,146 @@ +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import numpy as np + +import paddle +from paddle.base import core + + +def reduce_lr_on_plateau( + decay_rate, threshold, cooldown, patience, m, n, loss, var_list +): + def is_better(current, best, m, n): + if m == 'min' and n == 'rel': + return current < best - best * threshold + elif m == 'min' and n == 'abs': + return current < best - threshold + elif m == 'max' and n == 'rel': + return current > best + best * threshold + else: # mode == 'max' and epsilon_mode == 'abs': + return current > best + threshold + + if var_list[2] > 0: + var_list[2] -= 1 + return var_list[1] + + if is_better(loss, var_list[0], m, n): + var_list[0] = loss + var_list[3] = 0 + else: + var_list[3] += 1 + if var_list[3] > patience: + var_list[2] = cooldown + var_list[3] = 0 + new_lr = var_list[1] * decay_rate + var_list[1] = new_lr if var_list[1] - new_lr > 1e-8 else var_list[1] + + return var_list[1] + + +class TestReduceOnPlateauDecay(unittest.TestCase): + def test_ReduceLR(self): + # the decay rate must be less than 1.0 + with self.assertRaises(ValueError): + paddle.optimizer.lr.ReduceOnPlateau(learning_rate=1.0, factor=2.0) + # the mode must be "min" or "max" + with self.assertRaises(ValueError): + paddle.optimizer.lr.ReduceOnPlateau(learning_rate=1.0, mode="test") + # the threshold_mode must be "rel" or "abs" + with self.assertRaises(ValueError): + paddle.optimizer.lr.ReduceOnPlateau( + learning_rate=1.0, threshold_mode="test" + ) + with self.assertRaises(TypeError): + paddle.optimizer.lr.ReduceOnPlateau(learning_rate="test") + with self.assertRaises(TypeError): + paddle.optimizer.lr.ReduceOnPlateau(learning_rate=0.5).step("test") + + places = [paddle.CPUPlace()] + if core.is_compiled_with_cuda(): + places.append(paddle.CUDAPlace(0)) + + for place in places: + for m, n in zip( + ['min', 'max', 'min', 'max'], ['rel', 'rel', 'abs', 'abs'] + ): + kwargs = { + 'learning_rate': 1.0, + 'mode': m, + 'factor': 0.5, + 'patience': 3, + 'threshold': 1e-4, + 'threshold_mode': n, + 'cooldown': 1, + 'min_lr': 0, + 'epsilon': 1e-8, + 'verbose': False, + } + paddle.enable_static() + self._test_static(place, kwargs) + + def _test_static(self, place, kwargs): + paddle.enable_static() + + best = float("-10000") if kwargs['mode'] == "max" else float("10000") + current_lr = 1.0 + cooldown_counter = 0 + num_bad_epochs = 0 + var_list = [best, current_lr, cooldown_counter, num_bad_epochs] + + main_prog = paddle.static.Program() + start_prog = paddle.static.Program() + with paddle.static.program_guard(main_prog, start_prog): + x = paddle.pir.core.create_parameter( + 'float32', + [1], + name='x', + initializer=paddle.nn.initializer.ConstantInitializer( + value=float(1), force_cpu=False + ), + ) + paddle.increment(x) + loss = paddle.sin(x) + scheduler = paddle.optimizer.lr.ReduceOnPlateau(**kwargs) + adam = paddle.optimizer.Adam(learning_rate=scheduler) + adam.minimize(loss) + lr_var = adam._global_learning_rate() + # test_prog = main_prog.clone() + + exe = paddle.static.Executor(place) + exe.run(start_prog) + + for epoch in range(20): + for batch_id in range(1): + out, actual_lr = exe.run(main_prog, fetch_list=[loss, lr_var]) + expected_lr = reduce_lr_on_plateau( + kwargs['factor'], + kwargs['threshold'], + kwargs['cooldown'], + kwargs['patience'], + kwargs['mode'], + kwargs['threshold_mode'], + out[0], + var_list, + ) + + scheduler.step(out[0]) + actual_lr = scheduler() + self.assertEqual(actual_lr, np.array(expected_lr)) + + +if __name__ == '__main__': + unittest.main() From 137ead7cbab46ca6b2e904050852ee2d366ff310 Mon Sep 17 00:00:00 2001 From: Liujie0926 <44688141+Liujie0926@users.noreply.github.com> Date: Mon, 20 Nov 2023 14:50:31 +0800 Subject: [PATCH 21/46] [Auto_Parallel] Add pir case for CI (#59102) * add pir path * update --- tools/auto_parallel/ci_auto_parallel.sh | 59 ++++++++++++++++-------- tools/auto_parallel/target_path_lists.sh | 7 +++ 2 files changed, 47 insertions(+), 19 deletions(-) diff --git a/tools/auto_parallel/ci_auto_parallel.sh b/tools/auto_parallel/ci_auto_parallel.sh index bb4ad97a69b295..7516892db67ac2 100644 --- a/tools/auto_parallel/ci_auto_parallel.sh +++ b/tools/auto_parallel/ci_auto_parallel.sh @@ -21,7 +21,6 @@ mkdir -p /workspace/case_logs export log_path=/workspace/case_logs export case_list=() -# Insatll paddlepaddle-gpu install_paddle(){ echo -e "\033[31m ---- Install paddlepaddle-gpu \033" python -m pip install --user ${paddle} --force-reinstall --no-dependencies; @@ -30,7 +29,15 @@ install_paddle(){ get_diff_TO_case(){ cd ${paddle_dir} -let last_num=${#target_lists_for_hybrid_ci[@]}-1 +# get the location of "test/auto_parallel" in target_lists_for_hybrid_ci +count=0 +for element in "${target_lists_for_hybrid_ci[@]}";do + if [[ "$element" == "test/auto_parallel" ]]; then + test_num=$count + break + fi + count=$((count+1)) +done for file_name in `git diff --numstat upstream/${AGILE_COMPILE_BRANCH} |awk '{print $NF}'`;do arr_file_name=(${file_name//// }) dir1=${arr_file_name[0]} @@ -39,25 +46,32 @@ for file_name in `git diff --numstat upstream/${AGILE_COMPILE_BRANCH} |awk '{pri dir4=${arr_file_name[3]} file_item=$dir1/$dir2/$dir3/$dir4 echo "file_name:"${file_name}, "path:"${file_item} - if [ ! -f ${file_name} ];then # 针对pr删掉文件 + if [ ! -f ${file_name} ];then # deleting files for PR continue elif [[ ${file_name##*.} == "md" ]] || [[ ${file_name##*.} == "rst" ]] || [[ ${dir1} == "docs" ]];then continue else for ((i=0; i<${#target_lists_for_hybrid_ci[@]}; i++)); do - if [[ $i != ${last_num} ]] && [[ ${file_item} == *${target_lists_for_hybrid_ci[i]}* ]];then - case_list[${#case_list[*]}]=gpt-3 + if [[ $i != ${test_num} ]] && [[ ${file_item} == *${target_lists_for_hybrid_ci[i]}* ]];then + case_list[${#case_list[*]}]=gpt-3_auto case_list[${#case_list[*]}]=unit_test break - elif [[ $i == ${last_num} ]] && [[ ${file_item} == *${target_lists_for_hybrid_ci[i]}* ]];then + elif [[ $i == ${test_num} ]] && [[ ${file_item} == *${target_lists_for_hybrid_ci[i]}* ]];then case_list[${#case_list[*]}]=unit_test break else continue fi done + for ((i=0; i<${#target_lists_for_pir_ci[@]}; i++)); do + if [[ ${file_item} == *${target_lists_for_pir_ci[i]}* ]];then + case_list[${#case_list[*]}]=gpt-3_auto_pir + break + else + continue + fi + done fi - done } @@ -65,7 +79,7 @@ print_info(){ if [ $1 -ne 0 ];then EXCODE=2 if [ ! -f ${log_path}/$2 ];then - echo -e "\033[31m run CI FAIL \033" + echo -e "\033[31m run $2 CI FAIL \033" else mv ${log_path}/$2 ${log_path}/$2_FAIL.log echo -e "\033[31m ${log_path}/$2_FAIL \033" @@ -73,32 +87,39 @@ if [ $1 -ne 0 ];then fi exit $EXCODE else - echo -e "\033[32m run CI SUCCESS \033" + echo -e "\033[32m run $3 CI SUCCESS \033" fi } -get_diff_TO_case # 获取待执行case列表 -case_list=($(awk -v RS=' ' '!a[$1]++' <<< ${case_list[*]})) # 去重并将结果存储回原列表 +# Get the list of pending cases +get_diff_TO_case +# Remove duplicates and store the results back to the original list +case_list=($(awk -v RS=' ' '!a[$1]++' <<< ${case_list[*]})) if [[ ${#case_list[*]} -ne 0 ]];then echo -e "\033[31m =======CI Check case========= \033" echo -e "\033[31m ---- case_list length: ${#case_list[*]}, cases: ${case_list[*]} \033" + echo -e "\033[31m ============================= \033" set +e - echo -e "\033[31m ---- start run case \033" # Install paddle install_paddle case_num=1 + export FLAGS_before_hook=0 for case in ${case_list[*]};do echo -e "\033[31m ---- running case $case_num/${#case_list[*]}: ${case} \033" - if [[ ${case} == "gpt-3" ]];then - echo -e "\033[31m ---- running case gpt-3 auto \033" - bash /workspace/PaddleNLP/scripts/distribute/ci_case_auto.sh - print_info $? `ls -lt ${log_path} | grep gpt | head -n 1 | awk '{print $9}'` + if [[ ${case} == "gpt-3_auto" ]];then + bash /workspace/PaddleNLP/scripts/distribute/ci_case_auto.sh case_list_auto $FLAGS_before_hook + export FLAGS_before_hook=1 + print_info $? `ls -lt ${log_path} | grep "gpt" | grep -v "pir" | head -n 1 | awk '{print $9}'` ${case} + let case_num++ + elif [[ ${case} == "gpt-3_auto_pir" ]];then + bash /workspace/PaddleNLP/scripts/distribute/ci_case_auto.sh case_list_auto_pir $FLAGS_before_hook + export FLAGS_before_hook=1 + print_info $? `ls -lt ${log_path} | grep "pir" | head -n 1 | awk '{print $9}'` ${case} let case_num++ elif [[ ${case} == "unit_test" ]];then - echo -e "\033[31m ---- running case unit_test \033" bash /workspace/Paddle/tools/auto_parallel/ci_case_unit.sh - print_info $? `ls -lt ${log_path} | grep test | head -n 1 | awk '{print $9}'` + print_info $? `ls -lt ${log_path} | grep "test" | head -n 1 | awk '{print $9}'` ${case} let case_num++ else echo -e "\033[31m ---- no ${case} \033" @@ -110,7 +131,7 @@ if [[ ${#case_list[*]} -ne 0 ]];then if [ ! -f *FAIL* ];then FF=0 EXCODE=0 - echo -e "\033[32m ---- case Success \033" + echo -e "\033[32m ---- all case Success \033" else FF=`ls *FAIL*|wc -l` EXCODE=2 diff --git a/tools/auto_parallel/target_path_lists.sh b/tools/auto_parallel/target_path_lists.sh index 349941dbda3d2e..7ef0bf7d56d784 100644 --- a/tools/auto_parallel/target_path_lists.sh +++ b/tools/auto_parallel/target_path_lists.sh @@ -24,3 +24,10 @@ target_lists_for_hybrid_ci=( "paddle/phi/core/distributed" "test/auto_parallel" ) + +target_lists_for_pir_ci=( + "paddle/fluid/framework/new_executor" + "paddle/fluid/pir/dialect" + "paddle/fluid/pir/transforms" + "paddle/pir" +) From 0c0d1ec47a6526fc0a70024541654fdbc77d9181 Mon Sep 17 00:00:00 2001 From: Winters Montagne <118546135+WintersMontagne10335@users.noreply.github.com> Date: Mon, 20 Nov 2023 16:10:52 +0800 Subject: [PATCH 22/46] =?UTF-8?q?=E3=80=90PaddlePaddle=20Hackathon=205=20N?= =?UTF-8?q?o.48=E3=80=91Fix=20bug=20that=20thread=20configuration=20parame?= =?UTF-8?q?ters=20are=20out=20of=20bounds=20(#58307)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * modified: paddle/phi/kernels/gpu/strided_copy_kernel.cu * modified: VerifyThreadConfigurationParameters * fix bugs --- paddle/phi/kernels/gpu/strided_copy_kernel.cu | 2355 +++++++++-------- 1 file changed, 1249 insertions(+), 1106 deletions(-) diff --git a/paddle/phi/kernels/gpu/strided_copy_kernel.cu b/paddle/phi/kernels/gpu/strided_copy_kernel.cu index 65dae3fc89efe9..fc452eb44973dd 100644 --- a/paddle/phi/kernels/gpu/strided_copy_kernel.cu +++ b/paddle/phi/kernels/gpu/strided_copy_kernel.cu @@ -17,15 +17,358 @@ limitations under the License. */ #include "paddle/phi/core/kernel_registry.h" namespace phi { +bool VerifyStridedCopyThreadConfigurationParameters(const dim3& block, + const dim3& grid) { + return block.x <= 1024 && block.y <= 1024 && block.z <= 64 && + block.x * block.y * block.z <= 1024 && + block.x * block.y * block.z >= 96 && grid.y < 65536 && grid.z < 65536; +} -template -__global__ void StridedCopyFunc( +template +__global__ void StridedCopyCaseZeroFunc( const T* input_data, - phi::Array input_dims, phi::Array input_stride, T* output_data, - phi::Array output_dims, + phi::Array output_stride) { + int64_t input_offset = 0; + int64_t output_offset = 0; + float coordinate[6] = {threadIdx.x, + threadIdx.y, + threadIdx.z, + blockIdx.x, + blockIdx.y, + blockIdx.z}; + +#pragma unroll + for (int dim = RANK - 1; dim >= 0; --dim) { + input_offset += coordinate[RANK - 1 - dim] * input_stride[dim]; + output_offset += coordinate[RANK - 1 - dim] * output_stride[dim]; + } + + output_data[output_offset] = input_data[input_offset]; +} + +template +bool LaunchStridedCopyCazeZeroKernel( + const Context& dev_ctx, + const T* input_data, + const phi::Array& input_stride, + T* output_data, + const phi::Array& output_stride, + const phi::Array& dims, + int rank) { + if (rank > 6) { + return false; + } + + dim3 grid(1, 1, 1), block(1, 1, 1); + + if (rank >= 1) { + block.x = dims[rank - 1]; + } + + if (rank >= 2) { + block.y = dims[rank - 2]; + } + + if (rank >= 3) { + block.z = dims[rank - 3]; + } + + if (rank >= 4) { + grid.x = dims[rank - 4]; + } + + if (rank >= 5) { + grid.y = dims[rank - 5]; + } + + if (rank >= 6) { + grid.z = dims[rank - 6]; + } + + if (!VerifyStridedCopyThreadConfigurationParameters(block, grid)) { + return false; + } + + switch (rank) { + case 1: + StridedCopyCaseZeroFunc<<>>( + input_data, input_stride, output_data, output_stride); + break; + case 2: + StridedCopyCaseZeroFunc<<>>( + input_data, input_stride, output_data, output_stride); + break; + case 3: + StridedCopyCaseZeroFunc<<>>( + input_data, input_stride, output_data, output_stride); + break; + case 4: + StridedCopyCaseZeroFunc<<>>( + input_data, input_stride, output_data, output_stride); + break; + case 5: + StridedCopyCaseZeroFunc<<>>( + input_data, input_stride, output_data, output_stride); + break; + case 6: + StridedCopyCaseZeroFunc<<>>( + input_data, input_stride, output_data, output_stride); + break; + } + + return true; +} + +template +__global__ void StridedCopyCaseOneFunc( + const T* input_data, + phi::Array input_stride, + T* out_data, phi::Array output_stride, + phi::Array dims, + const int64_t x_max) { + int64_t x = blockIdx.x * blockDim.x + threadIdx.x; + if (x < x_max) { + int64_t input_offset = 0; + int64_t output_offset = 0; + + int64_t reg_dims[6] = { + dims[0], dims[1], dims[2], dims[3], dims[4], dims[5]}; + int64_t coordinate[phi::DDim::kMaxRank + 1]; + + switch (N) { + case 1: + coordinate[0] = x % reg_dims[0]; + break; + case 2: + coordinate[0] = x % reg_dims[0]; + coordinate[1] = x / reg_dims[0] % reg_dims[1]; + break; + case 3: + coordinate[0] = x % reg_dims[0]; + coordinate[1] = x / reg_dims[0] % reg_dims[1]; + coordinate[2] = x / (reg_dims[0] * reg_dims[1]); + break; + case 4: + coordinate[0] = x % reg_dims[0]; + coordinate[1] = x / reg_dims[0] % reg_dims[1]; + coordinate[2] = x / (reg_dims[0] * reg_dims[1]); + coordinate[3] = blockIdx.y % reg_dims[2]; + break; + case 5: + coordinate[0] = x % reg_dims[0]; + coordinate[1] = x / reg_dims[0] % reg_dims[1]; + coordinate[2] = x / (reg_dims[0] * reg_dims[1]); + coordinate[3] = blockIdx.y % reg_dims[2]; + coordinate[4] = blockIdx.y / reg_dims[2] % reg_dims[3]; + break; + case 6: + coordinate[0] = x % reg_dims[0]; + coordinate[1] = x / reg_dims[0] % reg_dims[1]; + coordinate[2] = x / (reg_dims[0] * reg_dims[1]); + coordinate[3] = blockIdx.y % reg_dims[2]; + coordinate[4] = blockIdx.y / reg_dims[2] % reg_dims[3]; + coordinate[5] = blockIdx.y / (reg_dims[2] * reg_dims[3]); + break; + case 7: + coordinate[0] = x % reg_dims[0]; + coordinate[1] = x / reg_dims[0] % reg_dims[1]; + coordinate[2] = x / (reg_dims[0] * reg_dims[1]); + coordinate[3] = blockIdx.y % reg_dims[2]; + coordinate[4] = blockIdx.y / reg_dims[2] % reg_dims[3]; + coordinate[5] = blockIdx.y / (reg_dims[2] * reg_dims[3]); + coordinate[6] = blockIdx.z % reg_dims[4]; + break; + case 8: + coordinate[0] = x % reg_dims[0]; + coordinate[1] = x / reg_dims[0] % reg_dims[1]; + coordinate[2] = x / (reg_dims[0] * reg_dims[1]); + coordinate[3] = blockIdx.y % reg_dims[2]; + coordinate[4] = blockIdx.y / reg_dims[2] % reg_dims[3]; + coordinate[5] = blockIdx.y / (reg_dims[2] * reg_dims[3]); + coordinate[6] = blockIdx.z % reg_dims[4]; + coordinate[7] = blockIdx.z / reg_dims[4] % reg_dims[5]; + break; + case 9: + coordinate[0] = x % reg_dims[0]; + coordinate[1] = x / reg_dims[0] % reg_dims[1]; + coordinate[2] = x / (reg_dims[0] * reg_dims[1]); + coordinate[3] = blockIdx.y % reg_dims[2]; + coordinate[4] = blockIdx.y / reg_dims[2] % reg_dims[3]; + coordinate[5] = blockIdx.y / (reg_dims[2] * reg_dims[3]); + coordinate[6] = blockIdx.z % reg_dims[4]; + coordinate[7] = blockIdx.z / reg_dims[4] % reg_dims[5]; + coordinate[8] = blockIdx.z / (reg_dims[4] * reg_dims[5]); + break; + } + +#pragma unroll + for (int dim = N - 1; dim >= 0; --dim) { + input_offset += coordinate[N - 1 - dim] * input_stride[dim]; + output_offset += coordinate[N - 1 - dim] * output_stride[dim]; + } + + out_data[output_offset] = input_data[input_offset]; + } +} + +template +bool LaunchStridedCopyCazeOneKernel( + const Context& dev_ctx, + const T* input_data, + const phi::Array& input_stride, + T* output_data, + const phi::Array& output_stride, + const phi::Array& dims, + int rank, + int numel) { + dim3 grid(1, 1, 1), block(1, 1, 1); + phi::Array cur_dims; + block.x = 512; + + if (rank >= 1) { + grid.x = (numel + block.x - 1) / block.x; + cur_dims[0] = dims[rank - 1]; + } + + if (rank >= 2) { + cur_dims[1] = dims[rank - 2]; + } + + if (rank >= 4) { + grid.x = (dims[rank - 1] * dims[rank - 2] * dims[rank - 3] + block.x - 1) / + block.x; + grid.y = dims[rank - 4]; + cur_dims[2] = dims[rank - 4]; + } + + if (rank >= 5) { + grid.y = dims[rank - 4] * dims[rank - 5]; + cur_dims[2] = dims[rank - 4]; + cur_dims[3] = dims[rank - 5]; + } + + if (rank >= 6) { + grid.y = dims[rank - 4] * dims[rank - 5] * dims[rank - 6]; + } + + if (rank >= 7) { + grid.z = dims[rank - 7]; + cur_dims[4] = dims[rank - 7]; + } + + if (rank >= 8) { + grid.z = dims[rank - 7] * dims[rank - 8]; + cur_dims[5] = dims[rank - 8]; + } + + if (rank >= 9) { + grid.z = dims[rank - 7] * dims[rank - 8] * dims[rank - 9]; + } + + if (!VerifyStridedCopyThreadConfigurationParameters(block, grid)) { + return false; + } + + switch (rank) { + case 1: + StridedCopyCaseOneFunc + <<>>(input_data, + input_stride, + output_data, + output_stride, + cur_dims, + dims[rank - 1]); + break; + case 2: + StridedCopyCaseOneFunc<<>>( + input_data, + input_stride, + output_data, + output_stride, + cur_dims, + dims[rank - 1] * dims[rank - 2]); + break; + case 3: + StridedCopyCaseOneFunc<<>>( + input_data, + input_stride, + output_data, + output_stride, + cur_dims, + dims[rank - 1] * dims[rank - 2] * dims[rank - 3]); + break; + case 4: + StridedCopyCaseOneFunc<<>>( + input_data, + input_stride, + output_data, + output_stride, + cur_dims, + dims[rank - 1] * dims[rank - 2] * dims[rank - 3]); + break; + case 5: + StridedCopyCaseOneFunc<<>>( + input_data, + input_stride, + output_data, + output_stride, + cur_dims, + dims[rank - 1] * dims[rank - 2] * dims[rank - 3]); + break; + case 6: + StridedCopyCaseOneFunc<<>>( + input_data, + input_stride, + output_data, + output_stride, + cur_dims, + dims[rank - 1] * dims[rank - 2] * dims[rank - 3]); + break; + case 7: + StridedCopyCaseOneFunc<<>>( + input_data, + input_stride, + output_data, + output_stride, + cur_dims, + dims[rank - 1] * dims[rank - 2] * dims[rank - 3]); + break; + case 8: + StridedCopyCaseOneFunc<<>>( + input_data, + input_stride, + output_data, + output_stride, + cur_dims, + dims[rank - 1] * dims[rank - 2] * dims[rank - 3]); + break; + case 9: + StridedCopyCaseOneFunc<<>>( + input_data, + input_stride, + output_data, + output_stride, + cur_dims, + dims[rank - 1] * dims[rank - 2] * dims[rank - 3]); + break; + default: + PADDLE_THROW(phi::errors::InvalidArgument( + "The rank of input should be less than 9, but received %d.", rank)); + } + + return true; +} + +template +__global__ void StridedCopyDefaultFunc( + const T* input_data, + phi::Array input_stride, + T* output_data, + phi::Array output_stride, + phi::Array dims, const int64_t numel) { int64_t gid = blockIdx.x * blockDim.x + threadIdx.x; #pragma unroll @@ -33,29 +376,412 @@ __global__ void StridedCopyFunc( int64_t input_offset = 0; int64_t index_tmp = i; #pragma unroll - for (int dim = IN_RANK - 1; dim >= 0; --dim) { - input_offset += (index_tmp % input_dims[dim]) * input_stride[dim]; - index_tmp = index_tmp / input_dims[dim]; + for (int dim = RANK - 1; dim >= 0; --dim) { + input_offset += (index_tmp % dims[dim]) * input_stride[dim]; + index_tmp = index_tmp / dims[dim]; } int64_t output_offset = 0; index_tmp = i; #pragma unroll - for (int dim = OUT_RANK - 1; dim >= 0; --dim) { - output_offset += (index_tmp % output_dims[dim]) * output_stride[dim]; - index_tmp = index_tmp / output_dims[dim]; + for (int dim = RANK - 1; dim >= 0; --dim) { + output_offset += (index_tmp % dims[dim]) * output_stride[dim]; + index_tmp = index_tmp / dims[dim]; } output_data[output_offset] = input_data[input_offset]; } } +template +void LaunchStridedCopyDefaultKernel( + const Context& dev_ctx, + const T* input_data, + const phi::Array& input_stride, + T* output_data, + const phi::Array& output_stride, + const phi::Array& dims, + int rank, + int numel) { + int64_t block = 512; + int64_t grid = (numel + block - 1) / block; + + switch (rank) { + case 1: + StridedCopyDefaultFunc<<>>( + input_data, input_stride, output_data, output_stride, dims, numel); + break; + case 2: + StridedCopyDefaultFunc<<>>( + input_data, input_stride, output_data, output_stride, dims, numel); + break; + case 3: + StridedCopyDefaultFunc<<>>( + input_data, input_stride, output_data, output_stride, dims, numel); + break; + case 4: + StridedCopyDefaultFunc<<>>( + input_data, input_stride, output_data, output_stride, dims, numel); + break; + case 5: + StridedCopyDefaultFunc<<>>( + input_data, input_stride, output_data, output_stride, dims, numel); + break; + case 6: + StridedCopyDefaultFunc<<>>( + input_data, input_stride, output_data, output_stride, dims, numel); + break; + case 7: + StridedCopyDefaultFunc<<>>( + input_data, input_stride, output_data, output_stride, dims, numel); + break; + case 8: + StridedCopyDefaultFunc<<>>( + input_data, input_stride, output_data, output_stride, dims, numel); + break; + case 9: + StridedCopyDefaultFunc<<>>( + input_data, input_stride, output_data, output_stride, dims, numel); + break; + default: + PADDLE_THROW(phi::errors::InvalidArgument( + "The rank of input should be less than 9, but received %d.", rank)); + } +} + +template +__global__ void Strided2ContiguousCaseZeroFunc( + const T* input_data, + phi::Array input_stride, + T* output_data) { + int64_t input_offset = 0; + int64_t output_offset = (blockIdx.z * gridDim.y * gridDim.x + + blockIdx.y * gridDim.x + blockIdx.x) * + blockDim.z * blockDim.y * blockDim.x + + threadIdx.z * blockDim.y * blockDim.x + + threadIdx.y * blockDim.x + threadIdx.x; + float coordinate[6] = {threadIdx.x, + threadIdx.y, + threadIdx.z, + blockIdx.x, + blockIdx.y, + blockIdx.z}; + +#pragma unroll + for (int dim = RANK - 1; dim >= 0; --dim) { + input_offset += coordinate[RANK - 1 - dim] * input_stride[dim]; + } + + output_data[output_offset] = input_data[input_offset]; +} + +template +bool LaunchStrided2ContiguousCazeZeroKernel( + const Context& dev_ctx, + const T* input_data, + const phi::Array& input_stride, + T* output_data, + const phi::Array& dims, + int rank) { + if (rank > 6) { + return false; + } + + dim3 grid(1, 1, 1), block(1, 1, 1); + + if (rank >= 1) { + block.x = dims[rank - 1]; + } + + if (rank >= 2) { + block.y = dims[rank - 2]; + } + + if (rank >= 3) { + block.z = dims[rank - 3]; + } + + if (rank >= 4) { + grid.x = dims[rank - 4]; + } + + if (rank >= 5) { + grid.y = dims[rank - 5]; + } + + if (rank >= 6) { + grid.z = dims[rank - 6]; + } + + if (!VerifyStridedCopyThreadConfigurationParameters(block, grid)) { + return false; + } + + switch (rank) { + case 1: + Strided2ContiguousCaseZeroFunc + <<>>( + input_data, input_stride, output_data); + break; + case 2: + Strided2ContiguousCaseZeroFunc + <<>>( + input_data, input_stride, output_data); + break; + case 3: + Strided2ContiguousCaseZeroFunc + <<>>( + input_data, input_stride, output_data); + break; + case 4: + Strided2ContiguousCaseZeroFunc + <<>>( + input_data, input_stride, output_data); + break; + case 5: + Strided2ContiguousCaseZeroFunc + <<>>( + input_data, input_stride, output_data); + break; + case 6: + Strided2ContiguousCaseZeroFunc + <<>>( + input_data, input_stride, output_data); + break; + } + + return true; +} + +template +__global__ void Strided2ContiguousCaseOneFunc( + const T* input_data, + phi::Array input_stride, + T* out_data, + phi::Array dims, + const int64_t x_max) { + int64_t x = blockIdx.x * blockDim.x + threadIdx.x; + if (x < x_max) { + int64_t input_offset = 0; + int64_t output_offset = (blockIdx.z * gridDim.y + blockIdx.y) * x_max + x; + + int64_t reg_dims[6] = { + dims[0], dims[1], dims[2], dims[3], dims[4], dims[5]}; + int64_t coordinate[phi::DDim::kMaxRank + 1]; + + switch (N) { + case 1: + coordinate[0] = x % reg_dims[0]; + break; + case 2: + coordinate[0] = x % reg_dims[0]; + coordinate[1] = x / reg_dims[0] % reg_dims[1]; + break; + case 3: + coordinate[0] = x % reg_dims[0]; + coordinate[1] = x / reg_dims[0] % reg_dims[1]; + coordinate[2] = x / (reg_dims[0] * reg_dims[1]); + break; + case 4: + coordinate[0] = x % reg_dims[0]; + coordinate[1] = x / reg_dims[0] % reg_dims[1]; + coordinate[2] = x / (reg_dims[0] * reg_dims[1]); + coordinate[3] = blockIdx.y % reg_dims[2]; + break; + case 5: + coordinate[0] = x % reg_dims[0]; + coordinate[1] = x / reg_dims[0] % reg_dims[1]; + coordinate[2] = x / (reg_dims[0] * reg_dims[1]); + coordinate[3] = blockIdx.y % reg_dims[2]; + coordinate[4] = blockIdx.y / reg_dims[2] % reg_dims[3]; + break; + case 6: + coordinate[0] = x % reg_dims[0]; + coordinate[1] = x / reg_dims[0] % reg_dims[1]; + coordinate[2] = x / (reg_dims[0] * reg_dims[1]); + coordinate[3] = blockIdx.y % reg_dims[2]; + coordinate[4] = blockIdx.y / reg_dims[2] % reg_dims[3]; + coordinate[5] = blockIdx.y / (reg_dims[2] * reg_dims[3]); + break; + case 7: + coordinate[0] = x % reg_dims[0]; + coordinate[1] = x / reg_dims[0] % reg_dims[1]; + coordinate[2] = x / (reg_dims[0] * reg_dims[1]); + coordinate[3] = blockIdx.y % reg_dims[2]; + coordinate[4] = blockIdx.y / reg_dims[2] % reg_dims[3]; + coordinate[5] = blockIdx.y / (reg_dims[2] * reg_dims[3]); + coordinate[6] = blockIdx.z % reg_dims[4]; + break; + case 8: + coordinate[0] = x % reg_dims[0]; + coordinate[1] = x / reg_dims[0] % reg_dims[1]; + coordinate[2] = x / (reg_dims[0] * reg_dims[1]); + coordinate[3] = blockIdx.y % reg_dims[2]; + coordinate[4] = blockIdx.y / reg_dims[2] % reg_dims[3]; + coordinate[5] = blockIdx.y / (reg_dims[2] * reg_dims[3]); + coordinate[6] = blockIdx.z % reg_dims[4]; + coordinate[7] = blockIdx.z / reg_dims[4] % reg_dims[5]; + break; + case 9: + coordinate[0] = x % reg_dims[0]; + coordinate[1] = x / reg_dims[0] % reg_dims[1]; + coordinate[2] = x / (reg_dims[0] * reg_dims[1]); + coordinate[3] = blockIdx.y % reg_dims[2]; + coordinate[4] = blockIdx.y / reg_dims[2] % reg_dims[3]; + coordinate[5] = blockIdx.y / (reg_dims[2] * reg_dims[3]); + coordinate[6] = blockIdx.z % reg_dims[4]; + coordinate[7] = blockIdx.z / reg_dims[4] % reg_dims[5]; + coordinate[8] = blockIdx.z / (reg_dims[4] * reg_dims[5]); + break; + } + +#pragma unroll + for (int dim = N - 1; dim >= 0; --dim) { + input_offset += coordinate[N - 1 - dim] * input_stride[dim]; + } + + out_data[output_offset] = input_data[input_offset]; + } +} + +template +bool LaunchStrided2ContiguousCazeOneKernel( + const Context& dev_ctx, + const T* input_data, + const phi::Array& input_stride, + T* output_data, + const phi::Array& dims, + int rank, + int numel) { + dim3 grid(1, 1, 1), block(1, 1, 1); + phi::Array cur_dims; + block.x = 512; + + if (rank >= 1) { + grid.x = (numel + block.x - 1) / block.x; + cur_dims[0] = dims[rank - 1]; + } + + if (rank >= 2) { + cur_dims[1] = dims[rank - 2]; + } + + if (rank >= 4) { + grid.x = (dims[rank - 1] * dims[rank - 2] * dims[rank - 3] + block.x - 1) / + block.x; + grid.y = dims[rank - 4]; + cur_dims[2] = dims[rank - 4]; + } + + if (rank >= 5) { + grid.y = dims[rank - 4] * dims[rank - 5]; + cur_dims[2] = dims[rank - 4]; + cur_dims[3] = dims[rank - 5]; + } + + if (rank >= 6) { + grid.y = dims[rank - 4] * dims[rank - 5] * dims[rank - 6]; + } + + if (rank >= 7) { + grid.z = dims[rank - 7]; + cur_dims[4] = dims[rank - 7]; + } + + if (rank >= 8) { + grid.z = dims[rank - 7] * dims[rank - 8]; + cur_dims[5] = dims[rank - 8]; + } + + if (rank >= 9) { + grid.z = dims[rank - 7] * dims[rank - 8] * dims[rank - 9]; + } + + if (!VerifyStridedCopyThreadConfigurationParameters(block, grid)) { + return false; + } + + switch (rank) { + case 1: + Strided2ContiguousCaseOneFunc<<>>( + input_data, input_stride, output_data, cur_dims, dims[rank - 1]); + break; + case 2: + Strided2ContiguousCaseOneFunc<<>>( + input_data, + input_stride, + output_data, + cur_dims, + dims[rank - 1] * dims[rank - 2]); + break; + case 3: + Strided2ContiguousCaseOneFunc<<>>( + input_data, + input_stride, + output_data, + cur_dims, + dims[rank - 1] * dims[rank - 2] * dims[rank - 3]); + break; + case 4: + Strided2ContiguousCaseOneFunc<<>>( + input_data, + input_stride, + output_data, + cur_dims, + dims[rank - 1] * dims[rank - 2] * dims[rank - 3]); + break; + case 5: + Strided2ContiguousCaseOneFunc<<>>( + input_data, + input_stride, + output_data, + cur_dims, + dims[rank - 1] * dims[rank - 2] * dims[rank - 3]); + break; + case 6: + Strided2ContiguousCaseOneFunc<<>>( + input_data, + input_stride, + output_data, + cur_dims, + dims[rank - 1] * dims[rank - 2] * dims[rank - 3]); + break; + case 7: + Strided2ContiguousCaseOneFunc<<>>( + input_data, + input_stride, + output_data, + cur_dims, + dims[rank - 1] * dims[rank - 2] * dims[rank - 3]); + break; + case 8: + Strided2ContiguousCaseOneFunc<<>>( + input_data, + input_stride, + output_data, + cur_dims, + dims[rank - 1] * dims[rank - 2] * dims[rank - 3]); + break; + case 9: + Strided2ContiguousCaseOneFunc<<>>( + input_data, + input_stride, + output_data, + cur_dims, + dims[rank - 1] * dims[rank - 2] * dims[rank - 3]); + break; + default: + PADDLE_THROW(phi::errors::InvalidArgument( + "The rank of input should be less than 9, but received %d.", rank)); + } + + return true; +} + template -__global__ void Strided2ContiguousFunc( +__global__ void Strided2ContiguousDefaultFunc( const T* input_data, - phi::Array input_dims, phi::Array input_stride, T* output_data, - phi::Array output_dims, - phi::Array output_stride, + phi::Array dims, const int64_t numel) { int64_t gid = blockIdx.x * blockDim.x + threadIdx.x; #pragma unroll @@ -64,21 +790,403 @@ __global__ void Strided2ContiguousFunc( int64_t index_tmp = i; #pragma unroll for (int dim = IN_RANK - 1; dim >= 0; --dim) { - input_offset += (index_tmp % input_dims[dim]) * input_stride[dim]; - index_tmp = index_tmp / input_dims[dim]; + input_offset += (index_tmp % dims[dim]) * input_stride[dim]; + index_tmp = index_tmp / dims[dim]; } output_data[i] = input_data[input_offset]; } } +template +void LaunchStrided2ContiguousDefaultKernel( + const Context& dev_ctx, + const T* input_data, + const phi::Array& input_stride, + T* output_data, + const phi::Array& dims, + int rank, + int numel) { + int64_t block = 512; + int64_t grid = (numel + block - 1) / block; + + switch (rank) { + case 1: + Strided2ContiguousDefaultFunc<<>>( + input_data, input_stride, output_data, dims, numel); + break; + case 2: + Strided2ContiguousDefaultFunc<<>>( + input_data, input_stride, output_data, dims, numel); + break; + case 3: + Strided2ContiguousDefaultFunc<<>>( + input_data, input_stride, output_data, dims, numel); + break; + case 4: + Strided2ContiguousDefaultFunc<<>>( + input_data, input_stride, output_data, dims, numel); + break; + case 5: + Strided2ContiguousDefaultFunc<<>>( + input_data, input_stride, output_data, dims, numel); + break; + case 6: + Strided2ContiguousDefaultFunc<<>>( + input_data, input_stride, output_data, dims, numel); + break; + case 7: + Strided2ContiguousDefaultFunc<<>>( + input_data, input_stride, output_data, dims, numel); + break; + case 8: + Strided2ContiguousDefaultFunc<<>>( + input_data, input_stride, output_data, dims, numel); + break; + case 9: + Strided2ContiguousDefaultFunc<<>>( + input_data, input_stride, output_data, dims, numel); + break; + default: + PADDLE_THROW(phi::errors::InvalidArgument( + "The rank of input should be less than 9, but received %d.", rank)); + } +} + +template +__global__ void Contiguous2StridedCaseZeroFunc( + const T* input_data, + T* output_data, + phi::Array output_stride) { + int64_t input_offset = (blockIdx.z * gridDim.y * gridDim.x + + blockIdx.y * gridDim.x + blockIdx.x) * + blockDim.z * blockDim.y * blockDim.x + + threadIdx.z * blockDim.y * blockDim.x + + threadIdx.y * blockDim.x + threadIdx.x; + int64_t output_offset = 0; + float coordinate[6] = {threadIdx.x, + threadIdx.y, + threadIdx.z, + blockIdx.x, + blockIdx.y, + blockIdx.z}; + +#pragma unroll + for (int dim = RANK - 1; dim >= 0; --dim) { + output_offset += coordinate[RANK - 1 - dim] * output_stride[dim]; + } + + output_data[output_offset] = input_data[input_offset]; +} + +template +bool LaunchContiguous2StridedCazeZeroKernel( + const Context& dev_ctx, + const T* input_data, + T* output_data, + const phi::Array& output_stride, + const phi::Array& dims, + int rank) { + if (rank > 6) { + return false; + } + + dim3 grid(1, 1, 1), block(1, 1, 1); + + if (rank >= 1) { + block.x = dims[rank - 1]; + } + + if (rank >= 2) { + block.y = dims[rank - 2]; + } + + if (rank >= 3) { + block.z = dims[rank - 3]; + } + + if (rank >= 4) { + grid.x = dims[rank - 4]; + } + + if (rank >= 5) { + grid.y = dims[rank - 5]; + } + + if (rank >= 6) { + grid.z = dims[rank - 6]; + } + + if (!VerifyStridedCopyThreadConfigurationParameters(block, grid)) { + return false; + } + + switch (rank) { + case 1: + Contiguous2StridedCaseZeroFunc + <<>>( + input_data, output_data, output_stride); + break; + case 2: + Contiguous2StridedCaseZeroFunc + <<>>( + input_data, output_data, output_stride); + break; + case 3: + Contiguous2StridedCaseZeroFunc + <<>>( + input_data, output_data, output_stride); + break; + case 4: + Contiguous2StridedCaseZeroFunc + <<>>( + input_data, output_data, output_stride); + break; + case 5: + Contiguous2StridedCaseZeroFunc + <<>>( + input_data, output_data, output_stride); + break; + case 6: + Contiguous2StridedCaseZeroFunc + <<>>( + input_data, output_data, output_stride); + break; + } + + return true; +} + +template +__global__ void Contiguous2StridedCaseOneFunc( + const T* input_data, + T* out_data, + phi::Array output_stride, + phi::Array dims, + const int64_t x_max) { + int64_t x = blockIdx.x * blockDim.x + threadIdx.x; + if (x < x_max) { + int64_t input_offset = (blockIdx.z * gridDim.y + blockIdx.y) * x_max + x; + int64_t output_offset = 0; + + int64_t reg_dims[6] = { + dims[0], dims[1], dims[2], dims[3], dims[4], dims[5]}; + int64_t coordinate[phi::DDim::kMaxRank + 1]; + + switch (N) { + case 1: + coordinate[0] = x % reg_dims[0]; + break; + case 2: + coordinate[0] = x % reg_dims[0]; + coordinate[1] = x / reg_dims[0] % reg_dims[1]; + break; + case 3: + coordinate[0] = x % reg_dims[0]; + coordinate[1] = x / reg_dims[0] % reg_dims[1]; + coordinate[2] = x / (reg_dims[0] * reg_dims[1]); + break; + case 4: + coordinate[0] = x % reg_dims[0]; + coordinate[1] = x / reg_dims[0] % reg_dims[1]; + coordinate[2] = x / (reg_dims[0] * reg_dims[1]); + coordinate[3] = blockIdx.y % reg_dims[2]; + break; + case 5: + coordinate[0] = x % reg_dims[0]; + coordinate[1] = x / reg_dims[0] % reg_dims[1]; + coordinate[2] = x / (reg_dims[0] * reg_dims[1]); + coordinate[3] = blockIdx.y % reg_dims[2]; + coordinate[4] = blockIdx.y / reg_dims[2] % reg_dims[3]; + break; + case 6: + coordinate[0] = x % reg_dims[0]; + coordinate[1] = x / reg_dims[0] % reg_dims[1]; + coordinate[2] = x / (reg_dims[0] * reg_dims[1]); + coordinate[3] = blockIdx.y % reg_dims[2]; + coordinate[4] = blockIdx.y / reg_dims[2] % reg_dims[3]; + coordinate[5] = blockIdx.y / (reg_dims[2] * reg_dims[3]); + break; + case 7: + coordinate[0] = x % reg_dims[0]; + coordinate[1] = x / reg_dims[0] % reg_dims[1]; + coordinate[2] = x / (reg_dims[0] * reg_dims[1]); + coordinate[3] = blockIdx.y % reg_dims[2]; + coordinate[4] = blockIdx.y / reg_dims[2] % reg_dims[3]; + coordinate[5] = blockIdx.y / (reg_dims[2] * reg_dims[3]); + coordinate[6] = blockIdx.z % reg_dims[4]; + break; + case 8: + coordinate[0] = x % reg_dims[0]; + coordinate[1] = x / reg_dims[0] % reg_dims[1]; + coordinate[2] = x / (reg_dims[0] * reg_dims[1]); + coordinate[3] = blockIdx.y % reg_dims[2]; + coordinate[4] = blockIdx.y / reg_dims[2] % reg_dims[3]; + coordinate[5] = blockIdx.y / (reg_dims[2] * reg_dims[3]); + coordinate[6] = blockIdx.z % reg_dims[4]; + coordinate[7] = blockIdx.z / reg_dims[4] % reg_dims[5]; + break; + case 9: + coordinate[0] = x % reg_dims[0]; + coordinate[1] = x / reg_dims[0] % reg_dims[1]; + coordinate[2] = x / (reg_dims[0] * reg_dims[1]); + coordinate[3] = blockIdx.y % reg_dims[2]; + coordinate[4] = blockIdx.y / reg_dims[2] % reg_dims[3]; + coordinate[5] = blockIdx.y / (reg_dims[2] * reg_dims[3]); + coordinate[6] = blockIdx.z % reg_dims[4]; + coordinate[7] = blockIdx.z / reg_dims[4] % reg_dims[5]; + coordinate[8] = blockIdx.z / (reg_dims[4] * reg_dims[5]); + break; + } + +#pragma unroll + for (int dim = N - 1; dim >= 0; --dim) { + output_offset += coordinate[N - 1 - dim] * output_stride[dim]; + } + + out_data[output_offset] = input_data[input_offset]; + } +} + +template +bool LaunchContiguous2StridedCazeOneKernel( + const Context& dev_ctx, + const T* input_data, + T* output_data, + const phi::Array& output_stride, + const phi::Array& dims, + int rank, + int numel) { + dim3 grid(1, 1, 1), block(1, 1, 1); + phi::Array cur_dims; + block.x = 512; + + if (rank >= 1) { + grid.x = (numel + block.x - 1) / block.x; + cur_dims[0] = dims[rank - 1]; + } + + if (rank >= 2) { + cur_dims[1] = dims[rank - 2]; + } + + if (rank >= 4) { + grid.x = (dims[rank - 1] * dims[rank - 2] * dims[rank - 3] + block.x - 1) / + block.x; + grid.y = dims[rank - 4]; + cur_dims[2] = dims[rank - 4]; + } + + if (rank >= 5) { + grid.y = dims[rank - 4] * dims[rank - 5]; + cur_dims[2] = dims[rank - 4]; + cur_dims[3] = dims[rank - 5]; + } + + if (rank >= 6) { + grid.y = dims[rank - 4] * dims[rank - 5] * dims[rank - 6]; + } + + if (rank >= 7) { + grid.z = dims[rank - 7]; + cur_dims[4] = dims[rank - 7]; + } + + if (rank >= 8) { + grid.z = dims[rank - 7] * dims[rank - 8]; + cur_dims[5] = dims[rank - 8]; + } + + if (rank >= 9) { + grid.z = dims[rank - 7] * dims[rank - 8] * dims[rank - 9]; + } + + if (!VerifyStridedCopyThreadConfigurationParameters(block, grid)) { + return false; + } + + switch (rank) { + case 1: + Contiguous2StridedCaseOneFunc<<>>( + input_data, output_data, output_stride, cur_dims, dims[rank - 1]); + break; + case 2: + Contiguous2StridedCaseOneFunc<<>>( + input_data, + output_data, + output_stride, + cur_dims, + dims[rank - 1] * dims[rank - 2]); + break; + case 3: + Contiguous2StridedCaseOneFunc<<>>( + input_data, + output_data, + output_stride, + cur_dims, + dims[rank - 1] * dims[rank - 2] * dims[rank - 3]); + break; + case 4: + Contiguous2StridedCaseOneFunc<<>>( + input_data, + output_data, + output_stride, + cur_dims, + dims[rank - 1] * dims[rank - 2] * dims[rank - 3]); + break; + case 5: + Contiguous2StridedCaseOneFunc<<>>( + input_data, + output_data, + output_stride, + cur_dims, + dims[rank - 1] * dims[rank - 2] * dims[rank - 3]); + break; + case 6: + Contiguous2StridedCaseOneFunc<<>>( + input_data, + output_data, + output_stride, + cur_dims, + dims[rank - 1] * dims[rank - 2] * dims[rank - 3]); + break; + case 7: + Contiguous2StridedCaseOneFunc<<>>( + input_data, + output_data, + output_stride, + cur_dims, + dims[rank - 1] * dims[rank - 2] * dims[rank - 3]); + break; + case 8: + Contiguous2StridedCaseOneFunc<<>>( + input_data, + output_data, + output_stride, + cur_dims, + dims[rank - 1] * dims[rank - 2] * dims[rank - 3]); + break; + case 9: + Contiguous2StridedCaseOneFunc<<>>( + input_data, + output_data, + output_stride, + cur_dims, + dims[rank - 1] * dims[rank - 2] * dims[rank - 3]); + break; + default: + PADDLE_THROW(phi::errors::InvalidArgument( + "The rank of input should be less than 9, but received %d.", rank)); + } + + return true; +} + template -__global__ void Contiguous2StridedFunc( +__global__ void Contiguous2StridedDefaultFunc( const T* input_data, - phi::Array input_dims, - phi::Array input_stride, T* output_data, - phi::Array output_dims, phi::Array output_stride, + phi::Array dims, const int64_t numel) { int64_t gid = blockIdx.x * blockDim.x + threadIdx.x; #pragma unroll @@ -87,13 +1195,68 @@ __global__ void Contiguous2StridedFunc( int64_t index_tmp = i; #pragma unroll for (int dim = OUT_RANK - 1; dim >= 0; --dim) { - output_offset += (index_tmp % output_dims[dim]) * output_stride[dim]; - index_tmp = index_tmp / output_dims[dim]; + output_offset += (index_tmp % dims[dim]) * output_stride[dim]; + index_tmp = index_tmp / dims[dim]; } output_data[output_offset] = input_data[i]; } } +template +void LaunchContiguous2StridedDefaultKernel( + const Context& dev_ctx, + const T* input_data, + T* output_data, + const phi::Array& output_stride, + const phi::Array& dims, + int rank, + int numel) { + int64_t block = 512; + int64_t grid = (numel + block - 1) / block; + + switch (rank) { + case 1: + Contiguous2StridedDefaultFunc<<>>( + input_data, output_data, output_stride, dims, numel); + break; + case 2: + Contiguous2StridedDefaultFunc<<>>( + input_data, output_data, output_stride, dims, numel); + break; + case 3: + Contiguous2StridedDefaultFunc<<>>( + input_data, output_data, output_stride, dims, numel); + break; + case 4: + Contiguous2StridedDefaultFunc<<>>( + input_data, output_data, output_stride, dims, numel); + break; + case 5: + Contiguous2StridedDefaultFunc<<>>( + input_data, output_data, output_stride, dims, numel); + break; + case 6: + Contiguous2StridedDefaultFunc<<>>( + input_data, output_data, output_stride, dims, numel); + break; + case 7: + Contiguous2StridedDefaultFunc<<>>( + input_data, output_data, output_stride, dims, numel); + break; + case 8: + Contiguous2StridedDefaultFunc<<>>( + input_data, output_data, output_stride, dims, numel); + break; + case 9: + Contiguous2StridedDefaultFunc<<>>( + input_data, output_data, output_stride, dims, numel); + break; + default: + PADDLE_THROW(phi::errors::InvalidArgument( + "The rank of input should be less than 9, but received %d.", rank)); + } +} + template void StridedCopyKernel(const Context& dev_ctx, const DenseTensor& input, @@ -122,9 +1285,9 @@ void StridedCopyKernel(const Context& dev_ctx, out->numel())); const T* input_data = input.data(); - int input_rank = input.dims().size(); - phi::Array input_stride; + int rank = input.dims().size(); phi::Array input_dims; + phi::Array input_stride; for (int i = 0; i < input.dims().size(); i++) { input_dims[i] = input.dims()[i]; input_stride[i] = input.strides()[i]; @@ -136,17 +1299,12 @@ void StridedCopyKernel(const Context& dev_ctx, "StridedCopyKernel's out tensor must complete " "mutable data before call kernel.")); - int output_rank = meta.dims.size(); phi::Array output_stride; - phi::Array output_dims; for (int i = 0; i < meta.dims.size(); i++) { - output_dims[i] = meta.dims[i]; output_stride[i] = meta.strides[i]; } auto numel = input.numel(); - int64_t block = 512; - int64_t grid = (numel + block - 1) / block; if (numel == 1) { #ifdef PADDLE_WITH_HIP @@ -165,1087 +1323,72 @@ void StridedCopyKernel(const Context& dev_ctx, } if (input.meta().is_contiguous()) { - switch (input_rank) { - case 1: - Contiguous2StridedFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 2: - Contiguous2StridedFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 3: - Contiguous2StridedFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 4: - Contiguous2StridedFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 5: - Contiguous2StridedFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 6: - Contiguous2StridedFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 7: - Contiguous2StridedFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 8: - Contiguous2StridedFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 9: - Contiguous2StridedFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - default: - PADDLE_THROW(phi::errors::InvalidArgument( - "The rank of input should be less than 9, but received %d.", - input_rank)); + if (LaunchContiguous2StridedCazeZeroKernel(dev_ctx, + input_data, + output_data, + output_stride, + input_dims, + rank)) { + } else if (LaunchContiguous2StridedCazeOneKernel(dev_ctx, + input_data, + output_data, + output_stride, + input_dims, + rank, + numel)) { + } else { + LaunchContiguous2StridedDefaultKernel(dev_ctx, + input_data, + output_data, + output_stride, + input_dims, + rank, + numel); } } else if (out->meta().is_contiguous()) { - switch (output_rank) { - case 1: - Strided2ContiguousFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 2: - Strided2ContiguousFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 3: - Strided2ContiguousFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 4: - Strided2ContiguousFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 5: - Strided2ContiguousFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 6: - Strided2ContiguousFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 7: - Strided2ContiguousFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 8: - Strided2ContiguousFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 9: - Strided2ContiguousFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - default: - PADDLE_THROW(phi::errors::InvalidArgument( - "The rank of output should be less than 9, but received %d.", - output_rank)); + if (LaunchStrided2ContiguousCazeZeroKernel( + dev_ctx, input_data, input_stride, output_data, input_dims, rank)) { + } else if (LaunchStrided2ContiguousCazeOneKernel(dev_ctx, + input_data, + input_stride, + output_data, + input_dims, + rank, + numel)) { + } else { + LaunchStrided2ContiguousDefaultKernel(dev_ctx, + input_data, + input_stride, + output_data, + input_dims, + rank, + numel); } } else { - switch (input_rank) { - case 1: { - switch (output_rank) { - case 1: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 2: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 3: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 4: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 5: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 6: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 7: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 8: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 9: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - default: - PADDLE_THROW(phi::errors::InvalidArgument( - "The rank of output should be less than 9, but received %d.", - output_rank)); - } - } break; - case 2: { - switch (output_rank) { - case 1: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 2: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 3: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 4: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 5: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 6: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 7: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 8: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 9: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - default: - PADDLE_THROW(phi::errors::InvalidArgument( - "The rank of output should be less than 9, but received %d.", - output_rank)); - } - } break; - case 3: { - switch (output_rank) { - case 1: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 2: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 3: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 4: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 5: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 6: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 7: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 8: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 9: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - default: - PADDLE_THROW(phi::errors::InvalidArgument( - "The rank of output should be less than 9, but received %d.", - output_rank)); - } - } break; - case 4: { - switch (output_rank) { - case 1: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 2: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 3: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 4: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 5: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 6: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 7: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 8: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 9: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - default: - PADDLE_THROW(phi::errors::InvalidArgument( - "The rank of output should be less than 9, but received %d.", - output_rank)); - } - } break; - case 5: { - switch (output_rank) { - case 1: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 2: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 3: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 4: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 5: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 6: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 7: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 8: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 9: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - default: - PADDLE_THROW(phi::errors::InvalidArgument( - "The rank of output should be less than 9, but received %d.", - output_rank)); - } - } break; - case 6: { - switch (output_rank) { - case 1: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 2: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 3: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 4: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 5: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 6: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 7: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 8: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 9: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - default: - PADDLE_THROW(phi::errors::InvalidArgument( - "The rank of output should be less than 9, but received %d.", - output_rank)); - } - } break; - case 7: { - switch (output_rank) { - case 1: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 2: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 3: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 4: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 5: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 6: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 7: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 8: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 9: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - default: - PADDLE_THROW(phi::errors::InvalidArgument( - "The rank of output should be less than 9, but received %d.", - output_rank)); - } - } break; - case 8: { - switch (output_rank) { - case 1: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 2: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 3: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 4: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 5: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 6: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 7: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 8: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 9: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - default: - PADDLE_THROW(phi::errors::InvalidArgument( - "The rank of output should be less than 9, but received %d.", - output_rank)); - } - } break; - case 9: { - switch (output_rank) { - case 1: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 2: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 3: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 4: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 5: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 6: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 7: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 8: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - case 9: - StridedCopyFunc - <<>>(input_data, - input_dims, - input_stride, - output_data, - output_dims, - output_stride, - numel); - break; - default: - PADDLE_THROW(phi::errors::InvalidArgument( - "The rank of output should be less than 9, but received %d.", - output_rank)); - } - } break; - default: - PADDLE_THROW(phi::errors::InvalidArgument( - "The rank of input should be less than 9, but received %d.", - input_rank)); + if (LaunchStridedCopyCazeZeroKernel(dev_ctx, + input_data, + input_stride, + output_data, + output_stride, + input_dims, + rank)) { + } else if (LaunchStridedCopyCazeOneKernel(dev_ctx, + input_data, + input_stride, + output_data, + output_stride, + input_dims, + rank, + numel)) { + } else { + LaunchStridedCopyDefaultKernel(dev_ctx, + input_data, + input_stride, + output_data, + output_stride, + input_dims, + rank, + numel); } } } From 7eade98ca1b6373a3964db7ba613ce2d74fee8e8 Mon Sep 17 00:00:00 2001 From: gouzil <66515297+gouzil@users.noreply.github.com> Date: Mon, 20 Nov 2023 16:20:35 +0800 Subject: [PATCH 23/46] [Dy2St] pir dy2st unittest verification - Part 7 (#59016) --- test/dygraph_to_static/test_closure_analysis.py | 16 ++++++++++------ .../test_gradient_aggregation.py | 7 +++++-- test/dygraph_to_static/test_jit_property_save.py | 9 ++++++++- test/dygraph_to_static/test_se_resnet.py | 8 +++++--- test/dygraph_to_static/test_warning.py | 1 - 5 files changed, 28 insertions(+), 13 deletions(-) diff --git a/test/dygraph_to_static/test_closure_analysis.py b/test/dygraph_to_static/test_closure_analysis.py index fe390108ed7d5a..16b407fadc1887 100644 --- a/test/dygraph_to_static/test_closure_analysis.py +++ b/test/dygraph_to_static/test_closure_analysis.py @@ -15,7 +15,10 @@ import inspect import unittest -from dygraph_to_static_utils_new import Dy2StTestBase, test_legacy_and_pir +from dygraph_to_static_utils_new import ( + Dy2StTestBase, + test_legacy_and_pir_exe_and_pir_api, +) from numpy import append import paddle @@ -194,6 +197,7 @@ def init_dygraph_func(self): {'func': set('i'), 'test_normal_argument': set('x')}, ] + @test_legacy_and_pir_exe_and_pir_api def test_main(self): if self.judge_type == 'push_pop_vars': for push_pop_vars, func in zip( @@ -260,7 +264,7 @@ def init_dygraph_func(self): class TestPushPopTrans(Dy2StTestBase): - @test_legacy_and_pir + @test_legacy_and_pir_exe_and_pir_api def test(self): def vlist_of_dict(x): ma = {'a': []} @@ -271,7 +275,7 @@ def vlist_of_dict(x): x = paddle.to_tensor([3]) print(paddle.jit.to_static(vlist_of_dict)(x)) - @test_legacy_and_pir + @test_legacy_and_pir_exe_and_pir_api def test2(self): import numpy as np @@ -284,7 +288,7 @@ def vlist_of_dict(x): x = paddle.to_tensor([3]) print(paddle.jit.to_static(vlist_of_dict)(x)) - @test_legacy_and_pir + @test_legacy_and_pir_exe_and_pir_api def test3(self): import numpy as np @@ -297,7 +301,7 @@ def vlist_of_dict(x): x = paddle.to_tensor([3]) print(paddle.jit.to_static(vlist_of_dict)(x)) - @test_legacy_and_pir + @test_legacy_and_pir_exe_and_pir_api def test4(self): import numpy as np @@ -310,7 +314,7 @@ def vlist_of_dict(x): x = paddle.to_tensor([3]) print(paddle.jit.to_static(vlist_of_dict)(x)) - @test_legacy_and_pir + @test_legacy_and_pir_exe_and_pir_api def test5(self): import numpy as np diff --git a/test/dygraph_to_static/test_gradient_aggregation.py b/test/dygraph_to_static/test_gradient_aggregation.py index 67b3ca8a987c73..06206dca5c4f91 100644 --- a/test/dygraph_to_static/test_gradient_aggregation.py +++ b/test/dygraph_to_static/test_gradient_aggregation.py @@ -15,7 +15,10 @@ import unittest import numpy as np -from dygraph_to_static_utils_new import Dy2StTestBase, test_legacy_and_pir +from dygraph_to_static_utils_new import ( + Dy2StTestBase, + test_legacy_and_pir_exe_and_pir_api, +) import paddle @@ -38,7 +41,7 @@ def forward(self, x): class TestGradientAggregationInDy2Static(Dy2StTestBase): - @test_legacy_and_pir + @test_legacy_and_pir_exe_and_pir_api def test_to_static(self): def simplenet_grad(inp, to_static=False): net = SimpleNet() diff --git a/test/dygraph_to_static/test_jit_property_save.py b/test/dygraph_to_static/test_jit_property_save.py index 6a254215fc8168..0cf994f34f95f1 100644 --- a/test/dygraph_to_static/test_jit_property_save.py +++ b/test/dygraph_to_static/test_jit_property_save.py @@ -14,7 +14,10 @@ import unittest -from dygraph_to_static_utils_new import Dy2StTestBase +from dygraph_to_static_utils_new import ( + Dy2StTestBase, + test_legacy_and_pir_exe_and_pir_api, +) import paddle @@ -33,18 +36,22 @@ def setUp(self): self.a = a self.b = b + @test_legacy_and_pir_exe_and_pir_api def test_property_save(self): self.assertEqual(self.a.get_float('a'), self.b.get_float('a')) self.assertEqual(self.a.get_float(0), 1.0) + @test_legacy_and_pir_exe_and_pir_api def test_size(self): self.assertEqual(self.b.size(), 2) self.assertEqual(self.a.size(), 2) + @test_legacy_and_pir_exe_and_pir_api def test_load_float(self): with self.assertRaises(ValueError): self.a.get_float(1) + @test_legacy_and_pir_exe_and_pir_api def test_set(self): """test property set.""" try: diff --git a/test/dygraph_to_static/test_se_resnet.py b/test/dygraph_to_static/test_se_resnet.py index f779babe69bb2c..b4b813e8ec9ea3 100644 --- a/test/dygraph_to_static/test_se_resnet.py +++ b/test/dygraph_to_static/test_se_resnet.py @@ -43,13 +43,15 @@ PRINT_STEP = 2 STEP_NUM = 10 -place = base.CUDAPlace(0) if base.is_compiled_with_cuda() else base.CPUPlace() +place = ( + paddle.CUDAPlace(0) if paddle.is_compiled_with_cuda() else paddle.CPUPlace() +) # Note: Set True to eliminate randomness. # 1. For one operation, cuDNN has several algorithms, # some algorithm results are non-deterministic, like convolution algorithms. -if base.is_compiled_with_cuda(): - base.set_flags({'FLAGS_cudnn_deterministic': True}) +if paddle.is_compiled_with_cuda(): + paddle.set_flags({'FLAGS_cudnn_deterministic': True}) train_parameters = { "learning_strategy": { diff --git a/test/dygraph_to_static/test_warning.py b/test/dygraph_to_static/test_warning.py index 2a80d113751561..ac4afdd8fce8e6 100644 --- a/test/dygraph_to_static/test_warning.py +++ b/test/dygraph_to_static/test_warning.py @@ -21,7 +21,6 @@ from paddle.static.nn import cond -@paddle.jit.to_static def fun1(): a = paddle.to_tensor(1) b = paddle.to_tensor(2) From 05210b0ab5c51117489c0e2aac9a7ea66e0ef846 Mon Sep 17 00:00:00 2001 From: gouzil <66515297+gouzil@users.noreply.github.com> Date: Mon, 20 Nov 2023 16:34:51 +0800 Subject: [PATCH 24/46] add `OpResult.clone()` (#59115) --- python/paddle/pir/math_op_patch.py | 26 ++++++++++++++++++++++ test/legacy_test/test_math_op_patch_pir.py | 17 ++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/python/paddle/pir/math_op_patch.py b/python/paddle/pir/math_op_patch.py index 21530fd636d79d..1ebd199fb4c9f2 100644 --- a/python/paddle/pir/math_op_patch.py +++ b/python/paddle/pir/math_op_patch.py @@ -331,6 +331,31 @@ def _size_(self): """ return paddle.numel(self) + def clone(self): + """ + Returns a new static OpResult, which is the clone of the original static + OpResult. It remains in the current graph, that is, the cloned OpResult + provides gradient propagation. Calling ``out = tensor.clone()`` is same + as ``out = assign(tensor)`` . + + Returns: + OpResult, The cloned OpResult. + + Examples: + .. code-block:: python + + >>> import paddle + + >>> paddle.enable_static() + + >>> # create a static OpResult + >>> x = paddle.static.data(name='x', shape=[3, 2, 1]) + >>> # create a cloned OpResult + >>> y = x.clone() + + """ + return paddle.assign(self) + import paddle opresult_methods = [ @@ -341,6 +366,7 @@ def _size_(self): ('ndim', _ndim), ('astype', astype), ('size', _size_), + ('clone', clone), ( '__add__', _binary_creator_('__add__', paddle.tensor.add, False, _scalar_add_), diff --git a/test/legacy_test/test_math_op_patch_pir.py b/test/legacy_test/test_math_op_patch_pir.py index e207fa74cb05b2..67a8794ed19683 100644 --- a/test/legacy_test/test_math_op_patch_pir.py +++ b/test/legacy_test/test_math_op_patch_pir.py @@ -427,6 +427,23 @@ def test_size(self): (output_x,) = exe.run(main_program, fetch_list=[x.size]) self.assertEqual(output_x, 24) + def test_clone(self): + x_np = np.random.random(size=[100, 10]).astype('float64') + with paddle.pir_utils.IrGuard(): + main_program, exe, program_guard = new_program() + with program_guard: + x = paddle.static.data( + name='x', shape=[100, 10], dtype="float64" + ) + a = x.clone() + (a_np,) = exe.run( + main_program, + feed={"x": x_np}, + fetch_list=[a], + ) + np.testing.assert_array_equal(x_np, a_np) + self.assertNotEqual(id(x), id(a)) + def test_math_exists(self): with paddle.pir_utils.IrGuard(): a = paddle.static.data(name='a', shape=[1], dtype='float32') From 2f5c5378d66a923ca315cc6b805f028bbe0410ac Mon Sep 17 00:00:00 2001 From: zhangbo9674 <82555433+zhangbo9674@users.noreply.github.com> Date: Mon, 20 Nov 2023 16:48:11 +0800 Subject: [PATCH 25/46] [PIR] Add parse kernel key interface (#59124) * add interface * add interface * add code * fix * fix * fix * fix * fix --- .../fluid/operators/generator/parse_utils.py | 7 +++ .../fluid/pir/dialect/op_generator/op_gen.py | 33 +++++++++- .../op_generator/parse_kernel_key_gen.py | 24 +++++++ .../dialect/operator/interface/interface.cc | 16 +++++ .../operator/interface/parse_kernel_key.h | 63 +++++++++++++++++++ .../pir/dialect/operator/ir/update_ops.yaml | 11 ++++ .../pir/transforms/pd_op_to_kernel_pass.cc | 13 +++- python/paddle/tensor/manipulation.py | 24 ++++++- 8 files changed, 187 insertions(+), 4 deletions(-) create mode 100644 paddle/fluid/pir/dialect/op_generator/parse_kernel_key_gen.py create mode 100644 paddle/fluid/pir/dialect/operator/interface/parse_kernel_key.h diff --git a/paddle/fluid/operators/generator/parse_utils.py b/paddle/fluid/operators/generator/parse_utils.py index 66a3ec8bdd1770..3395f265e26474 100644 --- a/paddle/fluid/operators/generator/parse_utils.py +++ b/paddle/fluid/operators/generator/parse_utils.py @@ -367,6 +367,7 @@ def check_op_config(op_entry, op_name): 'support_dygraph_mode', 'support_tensor', 'traits', + 'interfaces', ) infer_meta_key_set = ('func', 'param', 'spmd_rule') kernel_key_set = ( @@ -520,6 +521,11 @@ def parse_op_entry(op_entry: Dict[str, Any], name_field="op"): else: trait_list = [] + if "interfaces" in op_entry.keys(): + interface_list = parse_plain_list(op_entry["interfaces"]) + else: + interface_list = [] + op = { "name": op_name, "inputs": inputs, @@ -529,6 +535,7 @@ def parse_op_entry(op_entry: Dict[str, Any], name_field="op"): "data_transform": data_trans, "support_tensor": support_tensor, "traits": trait_list, + "interfaces": interface_list, } # op should be is_base_op or is_invoke_op or is_only_composite_op diff --git a/paddle/fluid/pir/dialect/op_generator/op_gen.py b/paddle/fluid/pir/dialect/op_generator/op_gen.py index abf2b5cd6cd1c0..27073c668fdfec 100644 --- a/paddle/fluid/pir/dialect/op_generator/op_gen.py +++ b/paddle/fluid/pir/dialect/op_generator/op_gen.py @@ -29,6 +29,7 @@ from op_kerneltype_gen import gen_kernel_type_for_var_str from op_member_func_gen import gen_op_get_inputs_outputs_str from op_verify_gen import gen_verify_func_str +from parse_kernel_key_gen import gen_parse_kernel_key_str from vjp_interface_black_list import vjp_interface_black_list # import from paddle/fluid/primitive/code_gen/gen.py @@ -61,6 +62,7 @@ #include "paddle/fluid/pir/dialect/operator/interface/op_yaml_info.h" #include "paddle/fluid/pir/dialect/operator/interface/infermeta.h" #include "paddle/fluid/pir/dialect/operator/interface/vjp.h" +#include "paddle/fluid/pir/dialect/operator/interface/parse_kernel_key.h" #include "paddle/fluid/pir/dialect/operator/interface/decomp.h" #include "paddle/fluid/pir/dialect/operator/trait/inplace.h" #include "paddle/fluid/pir/dialect/operator/trait/custom_vjp.h" @@ -103,6 +105,7 @@ class {op_name} : public pir::Op<{op_name}{interfaces}{traits}> {{ {build_mutable_attr_is_input_attr_num_over_1} void VerifySig(); {get_kernel_type_for_var_declare} +{parse_kernel_key_declare} {get_inputs_and_outputs} {exclusive_interface} }}; @@ -121,6 +124,10 @@ class {op_name} : public pir::Op<{op_name}{interfaces}{traits}> {{ const phi::DataType& expected_kernel_dtype); """ +parse_kernel_key_template = """ + static std::tuple ParseKernelKey(pir::Operation *op); +""" + # ===================================== # String Template for cc file code gen # ===================================== @@ -424,12 +431,21 @@ def __init__(self, op_yaml_item, op_compat_item): # parse traits list self.traits_list = self.parse_op_traits() + # parse interfaces list + self.interfaces_list = self.parse_op_interfaces() + def parse_op_traits(self): if 'traits' in self.op_yaml_item: return self.op_yaml_item['traits'] else: return [] + def parse_op_interfaces(self): + if 'interfaces' in self.op_yaml_item: + return self.op_yaml_item['interfaces'] + else: + return [] + def parse_forward_input_name(self): if 'forward' in self.op_yaml_item: forward_input_name_list = [] @@ -1133,8 +1149,9 @@ def OpGenerator( op_inplace_map = op_info.inplace_map op_view_map = op_info.view_map op_data_transform_map = op_info.data_transform_map - op_interfaces = ["paddle::dialect::OpYamlInfoInterface"] op_traits = op_info.traits_list + op_interfaces = op_info.interfaces_list + op_interfaces += ["paddle::dialect::OpYamlInfoInterface"] if op_info.infer_meta_func: op_interfaces += ["paddle::dialect::InferMetaInterface"] @@ -1251,6 +1268,10 @@ def OpGenerator( get_kernel_type_for_var_declare_template ) + parse_kernel_key_str = "" + if "paddle::dialect::ParseKernelKeyInterface" in op_interfaces: + parse_kernel_key_str = parse_kernel_key_template + if op_infer_meta_map is not None: ( build_args_with_muta_attr_not_input_for_declare, @@ -1384,6 +1405,7 @@ def OpGenerator( get_inputs_and_outputs=op_get_inputs_outputs_str, exclusive_interface=exclusive_interface_str, get_kernel_type_for_var_declare=get_kernel_type_for_var_declare_str, + parse_kernel_key_declare=parse_kernel_key_str, ) op_defined_str = "" else: @@ -1405,6 +1427,7 @@ def OpGenerator( get_inputs_and_outputs=op_get_inputs_outputs_str, exclusive_interface=exclusive_interface_str, get_kernel_type_for_var_declare=get_kernel_type_for_var_declare_str, + parse_kernel_key_declare=parse_kernel_key_str, ) attribute_names_str = ( '"' @@ -1576,6 +1599,13 @@ def OpGenerator( op_output_optional_list, ) + # generate op GetKernelKeyForVar function str + parse_kernel_key_define_str = '' + if "paddle::dialect::ParseKernelKeyInterface" in op_interfaces: + parse_kernel_key_define_str = gen_parse_kernel_key_str( + op_class_name + ) + # generate op GetKernelKeyForVar function str op_get_kernel_type_for_var_str = '' if dialect_name == "pd_op": @@ -1633,6 +1663,7 @@ def OpGenerator( ops_defined_list.append(op_verify_str) ops_defined_list.append(op_infer_meta_str) ops_defined_list.append(op_get_kernel_type_for_var_str) + ops_defined_list.append(parse_kernel_key_define_str) # NOTE(chenxi67)skip if dialect_name==cinn if dialect_name == "cinn": diff --git a/paddle/fluid/pir/dialect/op_generator/parse_kernel_key_gen.py b/paddle/fluid/pir/dialect/op_generator/parse_kernel_key_gen.py new file mode 100644 index 00000000000000..76a7c568170d79 --- /dev/null +++ b/paddle/fluid/pir/dialect/op_generator/parse_kernel_key_gen.py @@ -0,0 +1,24 @@ +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +OP_GET_KERNEL_TYPE_FOR_VAR_TEMPLATE = """ +std::tuple {op_name}::ParseKernelKey(pir::Operation *op) {{ + VLOG(4) << "Parse kernel key for op: {op_name}"; + return {op_name}ParseKernelKey(op); +}} +""" + + +def gen_parse_kernel_key_str(op_class_name): + return OP_GET_KERNEL_TYPE_FOR_VAR_TEMPLATE.format(op_name=op_class_name) diff --git a/paddle/fluid/pir/dialect/operator/interface/interface.cc b/paddle/fluid/pir/dialect/operator/interface/interface.cc index 01d8045425bea6..ae710d2c607eb5 100644 --- a/paddle/fluid/pir/dialect/operator/interface/interface.cc +++ b/paddle/fluid/pir/dialect/operator/interface/interface.cc @@ -16,7 +16,9 @@ #include "paddle/fluid/pir/dialect/operator/interface/get_kernel_type_for_var.h" #include "paddle/fluid/pir/dialect/operator/interface/infermeta.h" #include "paddle/fluid/pir/dialect/operator/interface/op_yaml_info.h" +#include "paddle/fluid/pir/dialect/operator/interface/parse_kernel_key.h" #include "paddle/fluid/pir/dialect/operator/interface/vjp.h" + namespace paddle { namespace dialect { std::vector> VjpInterface::Vjp( @@ -35,6 +37,19 @@ std::vector> VjpInterface::Vjp( } return impl_->vjp_(op, inputs, outputs, out_grads_value, stop_gradients); } + +KernelKeyTuple UniqueOpParseKernelKey(pir::Operation* op) { + DenseTensorType x_type = + op->operand_source(0).type().dyn_cast(); + phi::DataType dtype = TransToPhiDataType(x_type.dtype()); + pir::BoolAttribute is_sort = op->attribute("is_sorted"); + phi::Backend backend = phi::Backend::UNDEFINED; + if (is_sort.data()) { + backend = phi::Backend::CPU; + } + return {dtype, backend}; +} + } // namespace dialect } // namespace paddle @@ -43,3 +58,4 @@ IR_DEFINE_EXPLICIT_TYPE_ID(paddle::dialect::OpYamlInfoInterface) IR_DEFINE_EXPLICIT_TYPE_ID(paddle::dialect::VjpInterface) IR_DEFINE_EXPLICIT_TYPE_ID(paddle::dialect::DecompInterface) IR_DEFINE_EXPLICIT_TYPE_ID(paddle::dialect::GetKernelTypeForVarInterface) +IR_DEFINE_EXPLICIT_TYPE_ID(paddle::dialect::ParseKernelKeyInterface) diff --git a/paddle/fluid/pir/dialect/operator/interface/parse_kernel_key.h b/paddle/fluid/pir/dialect/operator/interface/parse_kernel_key.h new file mode 100644 index 00000000000000..80d407fcde1d94 --- /dev/null +++ b/paddle/fluid/pir/dialect/operator/interface/parse_kernel_key.h @@ -0,0 +1,63 @@ +// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "paddle/fluid/pir/dialect/operator/ir/op_type.h" +#include "paddle/fluid/pir/dialect/operator/utils/utils.h" +#include "paddle/phi/common/backend.h" +#include "paddle/phi/common/data_type.h" +#include "paddle/pir/core/op_base.h" + +using KernelKeyTuple = std::tuple; + +namespace paddle { +namespace dialect { +class ParseKernelKeyInterface + : public pir::OpInterfaceBase { + public: + struct Concept { + explicit Concept(KernelKeyTuple (*parse_kernel_key)(pir::Operation *op)) + : parse_kernel_key_(parse_kernel_key) {} + KernelKeyTuple (*parse_kernel_key_)(pir::Operation *op); + }; + + template + struct Model : public Concept { + static KernelKeyTuple ParseKernelKey(pir::Operation *op) { + return ConcreteOp::ParseKernelKey(op); + } + + Model() : Concept(ParseKernelKey) {} + }; + + /// Constructor + ParseKernelKeyInterface(pir::Operation *op, Concept *impl) + : pir::OpInterfaceBase(op), impl_(impl) {} + + KernelKeyTuple ParseKernelKey(pir::Operation *op) { + return impl_->parse_kernel_key_(op); + } + + private: + Concept *impl_; +}; + +// Register the ParseKernelKeyInterface for unique op. +KernelKeyTuple UniqueOpParseKernelKey(pir::Operation *op); + +} // namespace dialect +} // namespace paddle + +IR_DECLARE_EXPLICIT_TYPE_ID(paddle::dialect::ParseKernelKeyInterface) diff --git a/paddle/fluid/pir/dialect/operator/ir/update_ops.yaml b/paddle/fluid/pir/dialect/operator/ir/update_ops.yaml index de542e68f30b9d..d825016157ff73 100644 --- a/paddle/fluid/pir/dialect/operator/ir/update_ops.yaml +++ b/paddle/fluid/pir/dialect/operator/ir/update_ops.yaml @@ -12,3 +12,14 @@ data_type : dtype backend : place support_tensor : [start, end, step] + +- op : unique + args : (Tensor x, bool return_index=false, bool return_inverse=false, bool return_counts=false, int[] axis={}, DataType dtype=DataType::INT64, bool is_sorted=false) + output : Tensor(out), Tensor(indices), Tensor(inverse), Tensor(counts) + optional : indices, counts + infer_meta : + func : UniqueRawInferMeta + kernel : + func : unique + data_type : x + interfaces : paddle::dialect::ParseKernelKeyInterface diff --git a/paddle/fluid/pir/transforms/pd_op_to_kernel_pass.cc b/paddle/fluid/pir/transforms/pd_op_to_kernel_pass.cc index 495ac2602ce324..6f1ab3b705cb11 100644 --- a/paddle/fluid/pir/transforms/pd_op_to_kernel_pass.cc +++ b/paddle/fluid/pir/transforms/pd_op_to_kernel_pass.cc @@ -22,6 +22,7 @@ #include "paddle/fluid/pir/dialect/kernel/ir/kernel_op.h" #include "paddle/fluid/pir/dialect/kernel/ir/kernel_type.h" #include "paddle/fluid/pir/dialect/operator/interface/op_yaml_info.h" +#include "paddle/fluid/pir/dialect/operator/interface/parse_kernel_key.h" #include "paddle/fluid/pir/dialect/operator/ir/control_flow_op.h" #include "paddle/fluid/pir/dialect/operator/ir/manual_op.h" #include "paddle/fluid/pir/dialect/operator/ir/op_attribute.h" @@ -637,6 +638,15 @@ phi::KernelKey GetKernelKey( } } + // TODO(zhangbo): Add ParseKernelInterface + ParseKernelKeyInterface parse_kernel_key_interface = + op->dyn_cast(); + if (parse_kernel_key_interface) { + auto parsed_key = parse_kernel_key_interface.ParseKernelKey(op); + kernel_dtype = std::get<0>(parsed_key); + kernel_backend = std::get<1>(parsed_key); + } + if ((kernel_backend == phi::Backend::UNDEFINED || kernel_dtype == phi::DataType::UNDEFINED) && op->num_operands() > 0) { @@ -666,8 +676,7 @@ phi::KernelKey GetKernelKey( // don't know how to select the kernel in the next of op that // uses data op outout as inputs. So, we need set kernel backend // manually. - auto op_res = op->operand_source(i).dyn_cast(); - + auto op_res = input_tmp.dyn_cast(); if (!op_res) { continue; } diff --git a/python/paddle/tensor/manipulation.py b/python/paddle/tensor/manipulation.py index 685b10276c476f..a072d186bcece9 100644 --- a/python/paddle/tensor/manipulation.py +++ b/python/paddle/tensor/manipulation.py @@ -2670,7 +2670,7 @@ def unique( else: axis = [axis] attr_dtype = convert_np_dtype_to_dtype_(dtype) - if in_dynamic_or_pir_mode(): + if in_dynamic_mode(): out, indices, inverse, counts = _C_ops.unique( x, return_index, return_inverse, return_counts, axis, attr_dtype ) @@ -2685,6 +2685,28 @@ def unique( if len(outs) == 1: return outs[0] + return tuple(outs) + elif in_pir_mode(): + out, indices, inverse, counts = _C_ops.unique( + x, + return_index, + return_inverse, + return_counts, + axis, + attr_dtype, + True, + ) + outs = [out] + if return_index: + outs.append(indices) + if return_inverse: + outs.append(inverse) + if return_counts: + outs.append(counts) + + if len(outs) == 1: + return outs[0] + return tuple(outs) else: check_variable_and_dtype( From 988ae5a8f878c7c85162c2830b9998725340384c Mon Sep 17 00:00:00 2001 From: bukejiyu <52310069+bukejiyu@users.noreply.github.com> Date: Mon, 20 Nov 2023 17:44:40 +0800 Subject: [PATCH 26/46] [Inference] Add conv_fuse_pass, support conv2d+bn -> conv2d (#58724) * add conv2d_bn_fuse_pass --------- Co-authored-by: yxy --- .../fluid/inference/api/analysis_predictor.cc | 2 + .../pir/transforms/fusion/conv2d_fuse_pass.cc | 186 ++++++++++++++++++ .../pir/transforms/fusion/conv2d_fuse_pass.h | 26 +++ paddle/fluid/pybind/pir.cc | 1 + .../pattern_rewrite/pattern_rewrite_test.cc | 141 ++----------- test/ir/pir/fused_pass/pass_test.py | 76 +++++++ .../pir/fused_pass/test_conv2d_fuse_pass.py | 71 +++++++ 7 files changed, 376 insertions(+), 127 deletions(-) create mode 100644 paddle/fluid/pir/transforms/fusion/conv2d_fuse_pass.cc create mode 100644 paddle/fluid/pir/transforms/fusion/conv2d_fuse_pass.h create mode 100644 test/ir/pir/fused_pass/pass_test.py create mode 100644 test/ir/pir/fused_pass/test_conv2d_fuse_pass.py diff --git a/paddle/fluid/inference/api/analysis_predictor.cc b/paddle/fluid/inference/api/analysis_predictor.cc index 23054c01d8ef9c..5b143a5480db54 100644 --- a/paddle/fluid/inference/api/analysis_predictor.cc +++ b/paddle/fluid/inference/api/analysis_predictor.cc @@ -105,6 +105,7 @@ #include "paddle/fluid/ir_adaptor/translator/translate.h" #include "paddle/fluid/pir/transforms/constant_folding_pass.h" #include "paddle/fluid/pir/transforms/dead_code_elimination_pass.h" +#include "paddle/fluid/pir/transforms/fusion/conv2d_fuse_pass.h" #include "paddle/fluid/pir/transforms/inplace_pass.h" #include "paddle/fluid/pir/transforms/params_sync_among_devices_pass.h" #include "paddle/fluid/pir/transforms/pd_op_to_kernel_pass.h" @@ -772,6 +773,7 @@ bool AnalysisPredictor::PrepareExecutor() { ::pir::PassManager pm_for_op_program(::pir::IrContext::Instance(), 2); // TODO(liuyuanle): Uncomment constant_folding_pass after fix it // pm_for_op_program.AddPass(::pir::CreateConstantFoldingPass(sub_scope_)); + pm_for_op_program.AddPass(::pir::CreateConv2dFusePass()); pm_for_op_program.AddPass(::pir::CreateDeadCodeEliminationPass()); pm_for_op_program.AddPass( ::pir::CreateReplaceFetchWithShadowOutputPass()); diff --git a/paddle/fluid/pir/transforms/fusion/conv2d_fuse_pass.cc b/paddle/fluid/pir/transforms/fusion/conv2d_fuse_pass.cc new file mode 100644 index 00000000000000..be4e3930f7eaa0 --- /dev/null +++ b/paddle/fluid/pir/transforms/fusion/conv2d_fuse_pass.cc @@ -0,0 +1,186 @@ +// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/pir/pass/pass_registry.h" +#include "paddle/pir/pattern_rewrite/pattern_rewrite_driver.h" + +#include "paddle/fluid/pir/dialect/operator/ir/pd_op.h" +#include "paddle/fluid/pir/transforms/fusion/conv2d_fuse_pass.h" + +#include "paddle/fluid/pir/dialect/operator/ir/op_dialect.h" +#include "paddle/fluid/pir/dialect/operator/ir/op_type.h" +#include "paddle/fluid/pir/transforms/transform_general_functions.h" + +#include "paddle/phi/core/ddim.h" + +namespace { + +class Conv2dBnFusePattern + : public pir::OpRewritePattern { + public: + using pir::OpRewritePattern::OpRewritePattern; + bool MatchAndRewrite( + paddle::dialect::BatchNorm_Op op, + pir::PatternRewriter &rewriter) const override { // NOLINT + // The prev op should be conv2d op. + paddle::dialect::Conv2dOp conv2d_op = + pir::GetDefiningOpForInput(op, 0) + ->dyn_cast(); + if (!conv2d_op) return false; + + pir::OpResult conv2d_out = conv2d_op.out(); + if (!conv2d_out.HasOneUse()) return false; + + pir::Value conv2d_filter = conv2d_op.filter(); + + pir::OpResult conv2d_filter_result = + conv2d_filter.dyn_cast(); + IR_ENFORCE(conv2d_filter_result); + + pir::Value bn_input = op.x(); + IR_ENFORCE(bn_input == conv2d_out); + + pir::Value bn_mean = op.mean(); + pir::Value bn_variance = op.variance(); + pir::Value bn_scale = op.scale(); + pir::Value bn_bias = op.bias(); + + // --- deal with filter --- + rewriter.set_insertion_point(op); + phi::DDim bn_variance_shape = + bn_variance.type().dyn_cast().dims(); + float epsilon = op.attribute("epsilon").data(); + paddle::dialect::FullOp full_op = rewriter.Build( + phi::vectorize(bn_variance_shape), epsilon); + paddle::dialect::AddOp add_op = rewriter.Build( + bn_variance.dyn_cast(), full_op.out()); + paddle::dialect::SqrtOp sqrt_op = + rewriter.Build(add_op.out()); + paddle::dialect::DivideOp div_op = + rewriter.Build( + bn_scale.dyn_cast(), sqrt_op.out()); + // reshape scale + phi::DDim conv2d_filter_shape = pir::GetShapeFromValue(conv2d_filter); + phi::DDim bn_scale_shape = + bn_scale.type().dyn_cast().dims(); + std::vector bn_scale_new_shape(conv2d_filter_shape.size(), 1); + bn_scale_new_shape[0] = bn_scale_shape[0]; + paddle::dialect::ReshapeOp reshape_scale_op = + rewriter.Build(div_op.out(), + bn_scale_new_shape); + // new filter --> mul_op.out() + paddle::dialect::MultiplyOp mul_op = + rewriter.Build(conv2d_filter_result, + reshape_scale_op.out()); + + auto conv2d_attributes = conv2d_op->attributes(); + auto new_conv2d_op = rewriter.Build( + conv2d_op.input().dyn_cast(), + mul_op.out(), + conv2d_attributes); + + // --- deal with bias --- + paddle::dialect::MultiplyOp mul_bias_op = + rewriter.Build( + bn_mean.dyn_cast(), div_op.out()); + // new bias --> sub_op.out() + paddle::dialect::SubtractOp sub_op = + rewriter.Build( + bn_bias.dyn_cast(), mul_bias_op.out()); + // reshape new bias + phi::DDim new_conv2d_out_shape = + pir::GetShapeFromValue(new_conv2d_op.out()); + std::vector new_bias_new_shape(new_conv2d_out_shape.size(), 1); + std::string data_format = + new_conv2d_op.attribute("data_format").AsString(); + if (data_format != "NCHW") { + return false; + } + new_bias_new_shape[1] = new_conv2d_out_shape[1]; + paddle::dialect::ReshapeOp reshape_bias_op = + rewriter.Build(sub_op.out(), + new_bias_new_shape); + paddle::dialect::AddOp add_bias_op = rewriter.Build( + new_conv2d_op.out(), reshape_bias_op.out()); + + rewriter.ReplaceAllUsesWith(op.out(), add_bias_op.out()); + + rewriter.EraseOp(op); + rewriter.EraseOp(conv2d_op); + return true; + } +}; + +class BatchNormReplacePattern + : public pir::OpRewritePattern { + public: + using pir::OpRewritePattern::OpRewritePattern; + bool MatchAndRewrite( + paddle::dialect::BatchNormOp op, + pir::PatternRewriter &rewriter) const override { // NOLINT + auto bn_op = rewriter.Build( + op.x().dyn_cast(), + op.mean().dyn_cast(), + op.variance().dyn_cast(), + op.scale().dyn_cast(), + op.bias().dyn_cast(), + op->attributes()); + rewriter.ReplaceAllUsesWith(op.out(), bn_op.out()); + rewriter.EraseOp(op); + return true; + } +}; + +class Conv2dFusePass : public pir::PatternRewritePass { + public: + Conv2dFusePass() : pir::PatternRewritePass("conv2d_fuse_pass", 2) {} + + pir::RewritePatternSet InitializePatterns(pir::IrContext *context) override { + pir::RewritePatternSet ps(context); + auto conv_bn_pattern = std::make_unique( + context, + 1, + std::vector{paddle::dialect::FullOp::name(), + paddle::dialect::AddOp::name(), + paddle::dialect::SqrtOp::name(), + paddle::dialect::DivideOp::name(), + paddle::dialect::ReshapeOp::name(), + paddle::dialect::MultiplyOp::name(), + paddle::dialect::SubtractOp::name(), + paddle::dialect::Conv2dOp::name()}); + VLOG(4) << "Conv2dBnFusePattern will generate the following operations: "; + for (auto op_info : conv_bn_pattern->generated_ops()) { + VLOG(4) << "--- " << op_info.name(); + } + auto bn_replace_pattern = std::make_unique( + context, + 1, + std::vector{paddle::dialect::BatchNormOp::name()}); + ps.Add(std::move(bn_replace_pattern)); + ps.Add(std::move(conv_bn_pattern)); + return ps; + } +}; + +} // namespace + +namespace pir { + +std::unique_ptr CreateConv2dFusePass() { + return std::make_unique(); +} + +} // namespace pir + +REGISTER_IR_PASS(conv2d_fuse_pass, Conv2dFusePass); diff --git a/paddle/fluid/pir/transforms/fusion/conv2d_fuse_pass.h b/paddle/fluid/pir/transforms/fusion/conv2d_fuse_pass.h new file mode 100644 index 00000000000000..68d0ba98b0710b --- /dev/null +++ b/paddle/fluid/pir/transforms/fusion/conv2d_fuse_pass.h @@ -0,0 +1,26 @@ +// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include "paddle/pir/core/dll_decl.h" + +namespace pir { + +class Pass; + +IR_API std::unique_ptr CreateConv2dFusePass(); + +} // namespace pir diff --git a/paddle/fluid/pybind/pir.cc b/paddle/fluid/pybind/pir.cc index 4e8e954a312a85..e03f6972f5a87a 100644 --- a/paddle/fluid/pybind/pir.cc +++ b/paddle/fluid/pybind/pir.cc @@ -92,6 +92,7 @@ USE_PIR_PASS(fused_dropout_add_pass); USE_PIR_PASS(fused_linear_param_grad_add_pass); USE_PIR_PASS(inplace_pass); USE_PIR_PASS(replace_fetch_with_shadow_output_pass); +USE_PIR_PASS(conv2d_fuse_pass); PHI_DECLARE_bool(print_ir); diff --git a/test/cpp/pir/pattern_rewrite/pattern_rewrite_test.cc b/test/cpp/pir/pattern_rewrite/pattern_rewrite_test.cc index ea22c53f8bf911..fec08bf6ea47f4 100644 --- a/test/cpp/pir/pattern_rewrite/pattern_rewrite_test.cc +++ b/test/cpp/pir/pattern_rewrite/pattern_rewrite_test.cc @@ -55,6 +55,7 @@ // build Conv2dFusionOp #include "paddle/fluid/pir/dialect/operator/interface/infermeta.h" #include "paddle/fluid/pir/dialect/operator/interface/op_yaml_info.h" +#include "paddle/fluid/pir/transforms/fusion/conv2d_fuse_pass.h" #include "paddle/phi/api/lib/utils/allocator.h" #include "paddle/phi/infermeta/multiary.h" #include "paddle/pir/core/op_base.h" @@ -256,104 +257,6 @@ class RedundantTransposeFusePattern } }; -class Conv2dBnFusePattern - : public pir::OpRewritePattern { - public: - using pir::OpRewritePattern::OpRewritePattern; - bool MatchAndRewrite( - paddle::dialect::BatchNormOp op, - pir::PatternRewriter &rewriter) const override { // NOLINT - // The next op should be batch_norm. - paddle::dialect::Conv2dOp conv2d_op = - pir::GetDefiningOpForInput(op, 0) - ->dyn_cast(); - if (!conv2d_op) return false; - - pir::OpResult conv2d_out = conv2d_op.out(); - if (!conv2d_out.HasOneUse()) return false; - - pir::Value conv2d_filter = conv2d_op.filter(); - - // pir::GetParameterOp filter_parameter_op = - // conv2d_filter.GetDefiningOp()->dyn_cast(); - // if (!filter_parameter_op) return false; - - pir::OpResult conv2d_filter_result = - conv2d_filter.dyn_cast(); - IR_ENFORCE(conv2d_filter_result); - - pir::Value bn_input = op.x(); - IR_ENFORCE(bn_input == conv2d_out); - - pir::Value bn_mean = op.mean(); - pir::Value bn_variance = op.variance(); - pir::Value bn_scale = op.scale(); - pir::Value bn_bias = op.bias(); - - // --- deal with filter --- - rewriter.set_insertion_point(op); - phi::DDim bn_variance_shape = - bn_variance.type().dyn_cast().dims(); - float epsilon = op.attribute("epsilon").data(); - paddle::dialect::FullOp full_op = rewriter.Build( - phi::vectorize(bn_variance_shape), epsilon); - paddle::dialect::AddOp add_op = rewriter.Build( - bn_variance.dyn_cast(), full_op.out()); - paddle::dialect::SqrtOp sqrt_op = - rewriter.Build(add_op.out()); - paddle::dialect::DivideOp div_op = - rewriter.Build( - bn_scale.dyn_cast(), sqrt_op.out()); - // reshape scale - phi::DDim conv2d_filter_shape = pir::GetShapeFromValue(conv2d_filter); - phi::DDim bn_scale_shape = - bn_scale.type().dyn_cast().dims(); - std::vector bn_scale_new_shape(conv2d_filter_shape.size(), 1); - bn_scale_new_shape[0] = bn_scale_shape[0]; - paddle::dialect::ReshapeOp reshape_scale_op = - rewriter.Build(div_op.out(), - bn_scale_new_shape); - // new filter --> mul_op.out() - paddle::dialect::MultiplyOp mul_op = - rewriter.Build(conv2d_filter_result, - reshape_scale_op.out()); - - auto conv2d_attributes = conv2d_op->attributes(); - auto new_conv2d_op = rewriter.Build( - conv2d_op.input().dyn_cast(), - mul_op.out(), - conv2d_attributes); - - // --- deal with bias --- - paddle::dialect::MultiplyOp mul_bias_op = - rewriter.Build( - bn_mean.dyn_cast(), div_op.out()); - // new bias --> sub_op.out() - paddle::dialect::SubtractOp sub_op = - rewriter.Build( - bn_bias.dyn_cast(), mul_bias_op.out()); - // reshape new bias - phi::DDim new_conv2d_out_shape = - pir::GetShapeFromValue(new_conv2d_op.out()); - std::vector new_bias_new_shape(new_conv2d_out_shape.size(), 1); - std::string data_format = - new_conv2d_op.attribute("data_format").AsString(); - IR_ENFORCE(data_format == "NCHW", "Only support NCHW now."); - new_bias_new_shape[1] = new_conv2d_out_shape[1]; - paddle::dialect::ReshapeOp reshape_bias_op = - rewriter.Build(sub_op.out(), - new_bias_new_shape); - paddle::dialect::AddOp add_bias_op = rewriter.Build( - new_conv2d_op.out(), reshape_bias_op.out()); - - rewriter.ReplaceAllUsesWith(op.out(), add_bias_op.out()); - - rewriter.EraseOp(op); - rewriter.EraseOp(conv2d_op); - return true; - } -}; - namespace paddle { namespace dialect { class Conv2dFusionOpTest : public pir::Op(context); - auto conv_bn_pattern = std::make_unique( - context, - 1, - std::vector{paddle::dialect::FullOp::name(), - paddle::dialect::AddOp::name(), - paddle::dialect::SqrtOp::name(), - paddle::dialect::DivideOp::name(), - paddle::dialect::ReshapeOp::name(), - paddle::dialect::MultiplyOp::name(), - paddle::dialect::SubtractOp::name(), - paddle::dialect::Conv2dOp::name()}); - LOG(INFO) << "Conv2dBnFusePattern will generate the following operations: "; - for (auto op_info : conv_bn_pattern->generated_ops()) { - LOG(INFO) << "--- " << op_info.name(); - } - ps.Add(std::move(conv_bn_pattern)); - return ps; } }; @@ -1071,18 +957,18 @@ void BuildProgram(pir::Builder &builder) { // NOLINT builder.Build(full_input_op.out(), full_filter_op.out()); - paddle::dialect::BatchNormOp batch_norm_op = - builder.Build(conv2d_op.out(), - full_mean_op.out(), - full_variance_op.out(), - full_scale_op.out(), - full_bias_op.out(), - true, - 0.9, - 1e-6, - "NCHW", - false, - false); + paddle::dialect::BatchNorm_Op batch_norm_op = + builder.Build(conv2d_op.out(), + full_mean_op.out(), + full_variance_op.out(), + full_scale_op.out(), + full_bias_op.out(), + true, + 0.9, + 1e-6, + "NCHW", + false, + false); auto transpose1_op = builder.Build( batch_norm_op.out(), std::vector{0, 2, 3, 1}); @@ -1109,6 +995,7 @@ TEST(pattern_rewrite, Patterns) { paddle::framework::Scope scope; pir::PassManager pm(ctx); pm.AddPass(std::make_unique()); + pm.AddPass(pir::CreateConv2dFusePass()); pm.AddPass(pir::CreateConstantFoldingPass(&scope)); pm.AddPass(pir::CreateDeadCodeEliminationPass()); pm.EnablePassTiming(); diff --git a/test/ir/pir/fused_pass/pass_test.py b/test/ir/pir/fused_pass/pass_test.py new file mode 100644 index 00000000000000..5f7ca010d359c2 --- /dev/null +++ b/test/ir/pir/fused_pass/pass_test.py @@ -0,0 +1,76 @@ +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import paddle +from paddle import pir + + +class PassTest(unittest.TestCase): + @classmethod + def setUpClass(self): + self.main_program = paddle.static.Program() + self.feeds = None + self.fetch_list = None + self.valid_op_map = {} + self.pass_list = [] + self.pir_program = None + self.place_runtime = "cpu" + + def run_pir_pass(self): + if not isinstance(self.pass_list, list): + self.pass_list = [self.pass_list] + + pm = pir.PassManager() + for pass_name in self.pass_list: + pm.add_pass(pass_name) + + pm.run(self.pir_program) + + def check_fused_ops(self): + self.assertTrue( + len(self.valid_op_map) != 0, + "self.fuse_op_map cannot be empty!", + ) + op_names = [op.name() for op in self.pir_program.global_block().ops] + for valid_op_name, valid_op_count in self.valid_op_map.items(): + acctual_valid_op_count = op_names.count(valid_op_name) + self.assertTrue( + valid_op_count == acctual_valid_op_count, + "Checking of the number of fused operator < {} > failed. " + "Expected: {}, Received: {}".format( + valid_op_name, valid_op_count, acctual_valid_op_count + ), + ) + + def check_pass_correct(self, need_translate_to_pir=False, atol=1e-5): + self.assertTrue( + self.place_runtime == "cpu" or self.place_runtime == "gpu", + "The place param must be either GPU or CPU ", + ) + if self.place_runtime == "cpu": + executor = paddle.static.Executor(paddle.base.CPUPlace()) + elif self.place_runtime == "gpu": + executor = paddle.static.Executor(paddle.base.CUDAPlace(0)) + self.assertTrue( + need_translate_to_pir is False and self.pir_program is not None, + "using old ir need_translate_to_pir Cannot be fasle.\n \ + using new ir program Cannot be None. \n", + ) + if need_translate_to_pir and self.pir_program is None: + self.pir_program = pir.translate_to_pir(self.main_program.desc) + + self.run_pir_pass() + self.check_fused_ops() diff --git a/test/ir/pir/fused_pass/test_conv2d_fuse_pass.py b/test/ir/pir/fused_pass/test_conv2d_fuse_pass.py new file mode 100644 index 00000000000000..f925284ef7531e --- /dev/null +++ b/test/ir/pir/fused_pass/test_conv2d_fuse_pass.py @@ -0,0 +1,71 @@ +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +import numpy as np +from pass_test import PassTest + +import paddle + +paddle.enable_static() + + +@unittest.skipIf( + not paddle.base.core.is_compiled_with_cuda(), + "core is not complied with CUDA", +) +class TestConv2dFusePass(PassTest): + def build_ir_progam(self): + with paddle.pir_utils.IrGuard(): + self.pir_program = paddle.static.Program() + with paddle.pir.core.program_guard(self.pir_program): + x = paddle.static.data( + name='x', shape=[3, 1, 28, 28], dtype='float32' + ) + conv2d = paddle.nn.Conv2D( + in_channels=1, + out_channels=32, + kernel_size=3, + padding=1, + data_format='NCHW', + bias_attr=False, + ) + bn = paddle.nn.BatchNorm2D(num_features=32, data_format='NCHW') + out = bn(conv2d(x)) + + self.pass_list = ['conv2d_fuse_pass'] + self.feeds = {"x": np.random.random((3, 1, 28, 28)).astype("float32")} + self.fetch_list = [out] + self.valid_op_map = { + "pd_op.conv2d": 1, + "pd_op.batch_norm": 0, + } + + def setUp(self): + self.place_runtime = "gpu" + self.build_ir_progam() + + def test_check_output(self): + self.check_pass_correct() + + +class TestConv2dFusePassWtihCpu(TestConv2dFusePass): + def setUp(self): + self.place_runtime = "cpu" + self.build_ir_progam() + + +if __name__ == "__main__": + unittest.main() From 781f3bfedfac538da990fca0b703571837956699 Mon Sep 17 00:00:00 2001 From: cyber-pioneer <116002591+cyber-pioneer@users.noreply.github.com> Date: Mon, 20 Nov 2023 18:33:18 +0800 Subject: [PATCH 27/46] [Prim][PIR] Sink bn (#59054) * sink bn * fix template * fix code * bn manual * fix code * polish log * fix code * remove unused code * pause test case * fix test case --- .../decomp_interface_gen_op_list.py | 1 + .../dialect/operator/ir/manual_op_decomp.cc | 88 +++++++++++++ paddle/fluid/primitive/codegen/decomp_gen.py | 2 +- .../templates/decomp/generated_decomp.j2 | 19 ++- paddle/fluid/primitive/composite/composite.h | 116 ++++++++++++++++++ paddle/fluid/primitive/utils/utils.h | 8 ++ python/paddle/base/core.py | 8 +- python/paddle/decomposition/decomp.py | 40 +++--- .../test_batch_norm_op_prim_nchw.py | 9 ++ .../test_batch_norm_op_prim_nhwc.py | 2 + 10 files changed, 270 insertions(+), 23 deletions(-) diff --git a/paddle/fluid/pir/dialect/op_generator/decomp_interface_gen_op_list.py b/paddle/fluid/pir/dialect/op_generator/decomp_interface_gen_op_list.py index 2a8b43fc09ab50..a59855dcb265cd 100644 --- a/paddle/fluid/pir/dialect/op_generator/decomp_interface_gen_op_list.py +++ b/paddle/fluid/pir/dialect/op_generator/decomp_interface_gen_op_list.py @@ -19,6 +19,7 @@ # come into effect in generated file pd_op.h # manual decomp interface declare are located in manual_op.h decomp_interface_declare_gen_op_list = [ + "batch_norm", "mean", "squeeze", "add_n", diff --git a/paddle/fluid/pir/dialect/operator/ir/manual_op_decomp.cc b/paddle/fluid/pir/dialect/operator/ir/manual_op_decomp.cc index 43ffd4657c1fba..beca2f6f9b6409 100644 --- a/paddle/fluid/pir/dialect/operator/ir/manual_op_decomp.cc +++ b/paddle/fluid/pir/dialect/operator/ir/manual_op_decomp.cc @@ -29,5 +29,93 @@ namespace paddle { namespace dialect { using IntArray = paddle::experimental::IntArray; +std::vector> BatchNormOp::Decomp( + pir::Operation* op) { + VLOG(4) << "Decomp call batch_norm's decomp interface begin"; + BatchNormOp op_obj = op->dyn_cast(); + (void)op_obj; + + FLAGS_tensor_operants_mode = "static"; + + VLOG(6) << "Decomp Prepare inputs of batch_norm"; + + Tensor x(std::make_shared(op_obj.x())); + Tensor mean(std::make_shared(op_obj.mean())); + Tensor variance(std::make_shared(op_obj.variance())); + paddle::optional scale; + if (!IsEmptyValue(op_obj.scale())) { + scale = paddle::make_optional( + Tensor(std::make_shared(op_obj.scale()))); + } + paddle::optional bias; + if (!IsEmptyValue(op_obj.bias())) { + bias = paddle::make_optional( + Tensor(std::make_shared(op_obj.bias()))); + } + + VLOG(6) << "Decomp prepare attributes of batch_norm"; + bool is_test = op->attribute("is_test").dyn_cast().data(); + float momentum = + op->attribute("momentum").dyn_cast().data(); + float epsilon = + op->attribute("epsilon").dyn_cast().data(); + const std::string& data_layout = + op->attribute("data_layout").dyn_cast().AsString(); + bool use_global_stats = + op->attribute("use_global_stats").dyn_cast().data(); + bool trainable_statistics = op->attribute("trainable_statistics") + .dyn_cast() + .data(); + + VLOG(6) << "Decomp call batch_norm's forward composite rule prepare"; + + auto org_res = op->results(); + std::vector> res(org_res.size()); + + VLOG(6) << "Decomp call batch_norm's forward composite rule begin"; + + std::tuple op_res = + paddle::primitive::details::batch_norm_decomp( + x, + mean, + variance, + scale, + bias, + is_test, + momentum, + epsilon, + data_layout, + use_global_stats, + trainable_statistics); + + VLOG(6) << "Decomp call batch_norm's forward composite rule end"; + + res[0].push_back(std::static_pointer_cast( + std::get<0>(op_res).impl()) + ->value() + .dyn_cast()); + res[1].push_back(std::static_pointer_cast( + std::get<1>(op_res).impl()) + ->value() + .dyn_cast()); + res[2].push_back(std::static_pointer_cast( + std::get<2>(op_res).impl()) + ->value() + .dyn_cast()); + res[3].push_back(std::static_pointer_cast( + std::get<3>(op_res).impl()) + ->value() + .dyn_cast()); + res[4].push_back(std::static_pointer_cast( + std::get<4>(op_res).impl()) + ->value() + .dyn_cast()); + pir::OpResult reserve_space; + res[5].push_back(reserve_space); + + VLOG(4) << "Decomp call batch_norm's decomp interface end"; + return res; +} + } // namespace dialect } // namespace paddle diff --git a/paddle/fluid/primitive/codegen/decomp_gen.py b/paddle/fluid/primitive/codegen/decomp_gen.py index 06c78d38504de2..822456cc897182 100644 --- a/paddle/fluid/primitive/codegen/decomp_gen.py +++ b/paddle/fluid/primitive/codegen/decomp_gen.py @@ -197,7 +197,7 @@ def gen( for attr_item in item["attrs"]: if attr_item["typename"] not in attr_types_map.keys(): raise TypeError - attr_item["mapped_type"] = attr_types_map[attr_item["typename"]][0] + attr_item["mapped_type"] = attr_types_map[attr_item["typename"]] for out_item in item["outputs"]: if out_item["typename"] not in output_type_map.keys(): name = out_item["typename"] diff --git a/paddle/fluid/primitive/codegen/templates/decomp/generated_decomp.j2 b/paddle/fluid/primitive/codegen/templates/decomp/generated_decomp.j2 index 81c4a4937feed4..3efab61bf901a1 100644 --- a/paddle/fluid/primitive/codegen/templates/decomp/generated_decomp.j2 +++ b/paddle/fluid/primitive/codegen/templates/decomp/generated_decomp.j2 @@ -21,12 +21,14 @@ using IntArray = paddle::experimental::IntArray; {% set output_types=[] %} std::vector> {{class_name}}::Decomp(pir::Operation* op) { + VLOG(4) << "Decomp call {{fwd_name}}'s decomp interface begin"; + {{class_name}} op_obj = op->dyn_cast<{{class_name}}>(); (void)op_obj; FLAGS_tensor_operants_mode = "static"; - VLOG(4) << "Decomp Prepare inputs of {{fwd_name}}"; + VLOG(6) << "Decomp Prepare inputs of {{fwd_name}}"; {% for item in inputs -%} {% do input_names.append(item.name) %} @@ -66,7 +68,7 @@ std::vector> {{class_name}}::Decomp(pir::Operation* o {% endif %} {% endfor %} - VLOG(4) << "Decomp prepare attributes of {{fwd_name}}"; + VLOG(6) << "Decomp prepare attributes of {{fwd_name}}"; {% if attrs %} {% for item in attrs %} {% do attr_names.append(item.name) %} @@ -106,18 +108,25 @@ std::vector> {{class_name}}::Decomp(pir::Operation* o paddle::dialect::GetInt64Vector({{item.name}}_define_op->attribute("value"))); {% else %} - {{item.typename}} {{item.name}} = op->attribute("{{item.name}}").dyn_cast<{{item.mapped_type}}>().data(); + {% if item.mapped_type[0] == "pir::StrAttribute" %} + {{item.mapped_type[1]}} {{item.name}} = op->attribute("{{item.name}}").dyn_cast<{{item.mapped_type[0]}}>().AsString(); + {% else %} + {{item.mapped_type[1]}} {{item.name}} = op->attribute("{{item.name}}").dyn_cast<{{item.mapped_type[0]}}>().data(); + {% endif %} {% endif %} {% endfor %} {% endif %} - VLOG(4) << "Decomp prepare call {{fwd_name}}'s decomp interface"; + VLOG(6) << "Decomp call {{fwd_name}}'s forward composite rule prepare"; auto org_res = op->results(); std::vector> res(org_res.size()); + VLOG(6) << "Decomp call {{fwd_name}}'s forward composite rule begin"; {% if outputs|length == 1 %} Tensor op_res = paddle::primitive::details::{{fwd_name}}_decomp({{common.args(input_names, attr_names)}}); + VLOG(6) << "Decomp call {{fwd_name}}'s forward composite rule end"; + res[0].push_back( std::static_pointer_cast(op_res.impl()) ->value() @@ -129,6 +138,7 @@ std::vector> {{class_name}}::Decomp(pir::Operation* o {% endfor %} std::tuple<{{common.sequence('', '', ', ', output_types)}}> op_res = paddle::primitive::details::{{fwd_name}}_decomp( {{common.args(input_names, attr_names)}}); + VLOG(6) << "Decomp call {{fwd_name}}'s forward composite rule end"; {% for k in range(outputs|length) %} {% if outputs[k].intermediate and fwd_name in decomp_ops_list_contain_unused_output %} pir::OpResult {{outputs[k].name}}; @@ -139,6 +149,7 @@ std::vector> {{class_name}}::Decomp(pir::Operation* o {% endfor %} {% endif %} + VLOG(4) << "Decomp call {{fwd_name}}'s decomp interface end"; return res; } {% endmacro %} diff --git a/paddle/fluid/primitive/composite/composite.h b/paddle/fluid/primitive/composite/composite.h index 9a352d74d4d3f4..572834437a4b5f 100644 --- a/paddle/fluid/primitive/composite/composite.h +++ b/paddle/fluid/primitive/composite/composite.h @@ -62,6 +62,122 @@ Tensor mean_decomp(const Tensor& x, const IntArray& axis, bool keepdim) { } } +template +std::tuple batch_norm_decomp( + const Tensor& x, + const Tensor& run_mean, + const Tensor& run_var, + const paddle::optional& scale, + const paddle::optional& bias, + bool is_test, + float momentum, + float epsilon, + const std::string& data_layout, + bool use_global_stats, + bool trainable_statistics) { + auto org_dtype = x.dtype(); + Tensor x_cast = x; + + bool need_cast = org_dtype == phi::DataType::FLOAT16 || + org_dtype == phi::DataType::BFLOAT16; + + if (need_cast) { + x_cast = cast(x, phi::DataType::FLOAT32); + } + + std::vector x_dim = phi::vectorize(x_cast.dims()); + int rank = x_dim.size(); + DataLayout data_layout_ = phi::StringToDataLayout(data_layout); + int feature_axis; + if (data_layout_ == DataLayout::kNCHW) { + feature_axis = 1; + } else if (data_layout_ == DataLayout::kNHWC) { + feature_axis = rank - 1; + } else { + PADDLE_THROW( + phi::errors::InvalidArgument("Unknown storage order: %s", data_layout)); + } + std::vector reduce_axes; + for (int i = 0; i < rank; ++i) { + if (i != feature_axis) { + reduce_axes.push_back(i); + } + } + std::vector stats_shape; + for (int i = 0; i < rank; ++i) { + if (find_value(reduce_axes, i) == false) { + stats_shape.push_back(x_dim[i]); + } else { + stats_shape.push_back(1); + } + } + + Tensor half = full(IntArray({1}), -0.5, x_cast.dtype()); + + bool use_run_stat = (is_test && (!trainable_statistics)) || use_global_stats; + Tensor x_hat; + Tensor batch_mean; + Tensor inv_std; + Tensor run_mean_; + Tensor run_var_; + if (!use_run_stat) { + batch_mean = mean_decomp(x_cast, IntArray(reduce_axes), false); + auto temp = mean_decomp(x_cast * x_cast, IntArray(reduce_axes), false); + auto batch_var = temp - batch_mean * batch_mean; + inv_std = elementwise_pow((batch_var + epsilon), half); + if (data_layout_ == DataLayout::kNHWC) { + x_hat = (x_cast - batch_mean) * inv_std; + } else { + x_hat = (x_cast - reshape(batch_mean, stats_shape)) * + reshape(inv_std, stats_shape); + } + run_mean_ = run_mean * momentum + batch_mean * (1. - momentum); + run_var_ = run_var * momentum + batch_var * (1. - momentum); + } else { + batch_mean = full(phi::vectorize(run_mean.dims()), 0, run_mean.dtype()); + auto batch_var = + full(phi::vectorize(run_var.dims()), 0, run_var.dtype()); + inv_std = elementwise_pow((batch_var + epsilon), half); + if (data_layout_ == DataLayout::kNHWC) { + x_hat = + (x_cast - run_mean) * elementwise_pow((run_var + epsilon), half); + } else { + x_hat = (x_cast - reshape(run_mean, stats_shape)) * + elementwise_pow((reshape(run_var, stats_shape) + epsilon), + half); + } + run_mean_ = assign(run_mean); + run_var_ = assign(run_var); + } + Tensor y; + Tensor new_scale = + scale ? scale.get() + : full(phi::vectorize(x_cast.dims()), 1, x_cast.dtype()); + Tensor new_bias = + bias ? bias.get() + : full(phi::vectorize(x_cast.dims()), 0, x_cast.dtype()); + if (data_layout_ == DataLayout::kNHWC) { + y = x_hat * new_scale + new_bias; + } else { + y = x_hat * reshape(new_scale, stats_shape) + + reshape(new_bias, stats_shape); + } + if (need_cast) { + y = cast(y, org_dtype); + } + Tensor reserve_space; + + auto batch_mean_ = assign(batch_mean); + auto inv_std_ = assign(inv_std); + if (!use_run_stat) { + return std::make_tuple( + y, run_mean_, run_var_, batch_mean_, inv_std_, reserve_space); + } else { + return std::make_tuple( + y, run_mean_, run_var_, batch_mean_, inv_std_, reserve_space); + } +} + template Tensor softmax_decomp(const Tensor& x, const int& axis) { auto org_dtype = x.dtype(); diff --git a/paddle/fluid/primitive/utils/utils.h b/paddle/fluid/primitive/utils/utils.h index da6cd28bfa476b..7c3c9a163fae08 100644 --- a/paddle/fluid/primitive/utils/utils.h +++ b/paddle/fluid/primitive/utils/utils.h @@ -139,5 +139,13 @@ std::vector> ConstructVjpResultByStopGradients( const std::vector>& outputs, const std::vector>& stop_gradients); +static bool find_value(const std::vector& vec, int64_t value) { + if (std::find(vec.begin(), vec.end(), value) != vec.end()) { + return true; + } else { + return false; + } +} + } // namespace primitive } // namespace paddle diff --git a/python/paddle/base/core.py b/python/paddle/base/core.py index fb98aaadab21bd..c491d90c43a919 100644 --- a/python/paddle/base/core.py +++ b/python/paddle/base/core.py @@ -503,8 +503,14 @@ def _get_batch_norm_none_var(op): "unsqueeze2": ["XShape"], } + # some intermediate outputs like xshape will no longer used after decomp, but return none to keep output num the same as origin op -decomp_ops_list_contain_unused_output = ["pd_op.squeeze", "pd_op.unsqueeze"] +# key is the name of op, and value is the index of output in op.outputs +decomp_ops_contain_unused_output = { + "pd_op.squeeze": [1], + "pd_op.unsqueeze": [1], + "pd_op.batch_norm": [5], +} def _set_prim_forward_blacklist(*args): diff --git a/python/paddle/decomposition/decomp.py b/python/paddle/decomposition/decomp.py index ea27e9ab7924a5..adc4b89ca70510 100644 --- a/python/paddle/decomposition/decomp.py +++ b/python/paddle/decomposition/decomp.py @@ -19,7 +19,7 @@ from paddle.autograd import ir_backward from paddle.base.core import ( call_decomp, - decomp_ops_list_contain_unused_output, + decomp_ops_contain_unused_output, has_decomp, ) from paddle.base.libpaddle.pir import Block, Operation, Program @@ -37,22 +37,20 @@ def _build_tensor_tuple(xs): def _analyse_decomp_results(orig_outs, decomp_outs, op): - intermediate_status = op.get_output_intermediate_status() - assert len(orig_outs) == len(decomp_outs) == len(intermediate_status) + assert len(orig_outs) == len(decomp_outs) res = [] - for org_item, new_item, value in zip( - orig_outs, decomp_outs, intermediate_status - ): - if isinstance(org_item, pir.OpResult): - if value and op.name() in decomp_ops_list_contain_unused_output: - assert new_item[0] is None + for idx, value in enumerate(decomp_outs): + if isinstance(orig_outs[idx], pir.OpResult): + if ( + op.name() in decomp_ops_contain_unused_output.keys() + and idx in decomp_ops_contain_unused_output[op.name()] + ): + assert value[0] is None else: - assert len(new_item) == 1 and isinstance( - new_item[0], pir.OpResult - ) - res.append(new_item[0]) + assert len(value) == 1 and isinstance(value[0], pir.OpResult) + res.append(value[0]) else: - res.append(new_item) + res.append(value) return res @@ -278,10 +276,18 @@ def _decompose_subgraph(block, orig_vars, dst_vars, op_filter): _check_op_results( op_name, orig_outs, new_outs, orig_vars, dst_vars ) - if op.name() in decomp_ops_list_contain_unused_output: - orig_outs[0].replace_all_uses_with(new_outs[0]) + if op.name() in decomp_ops_contain_unused_output.keys(): + for idx in range(len(orig_outs)): + if ( + idx + not in decomp_ops_contain_unused_output[op.name()] + ): + orig_outs[idx].replace_all_uses_with(new_outs[idx]) else: - op.replace_all_uses_with(new_outs) + if op.name() in decomp_ops_contain_unused_output.keys(): + orig_outs[0].replace_all_uses_with(new_outs[0]) + else: + op.replace_all_uses_with(new_outs) block.remove_op(op) if temp_op is not None: diff --git a/test/legacy_test/test_batch_norm_op_prim_nchw.py b/test/legacy_test/test_batch_norm_op_prim_nchw.py index 88d02468e04000..0144a33eed1d02 100644 --- a/test/legacy_test/test_batch_norm_op_prim_nchw.py +++ b/test/legacy_test/test_batch_norm_op_prim_nchw.py @@ -64,6 +64,7 @@ def setUp(self): self.op_type = "batch_norm" self.prim_op_type = "comp" self.python_out_sig = ["Y"] + self.check_prim_pir = True self.initConfig() self.initTestCase() @@ -74,6 +75,7 @@ def test_check_output(self): no_check_set=None, check_prim=True, only_check_prim=True, + check_prim_pir=self.check_prim_pir, ) if paddle.is_compiled_with_cuda(): self.check_output_with_place( @@ -81,6 +83,7 @@ def test_check_output(self): no_check_set=None, check_prim=True, only_check_prim=True, + check_prim_pir=self.check_prim_pir, ) def test_check_grad_x(self): @@ -92,6 +95,7 @@ def test_check_grad_x(self): user_defined_grad_outputs=self.out_grad, check_prim=True, only_check_prim=True, + check_prim_pir=self.check_prim_pir, ) if paddle.is_compiled_with_cuda(): self.check_grad_with_place( @@ -101,6 +105,7 @@ def test_check_grad_x(self): user_defined_grad_outputs=self.out_grad, check_prim=True, only_check_prim=True, + check_prim_pir=self.check_prim_pir, ) def test_check_grad_scale_bias(self): @@ -124,6 +129,7 @@ def test_check_grad_scale_bias(self): user_defined_grad_outputs=self.out_grad, check_prim=True, only_check_prim=True, + check_prim_pir=self.check_prim_pir, ) if paddle.is_compiled_with_cuda(): self.check_grad_with_place( @@ -133,6 +139,7 @@ def test_check_grad_scale_bias(self): user_defined_grad_outputs=self.out_grad, check_prim=True, only_check_prim=True, + check_prim_pir=self.check_prim_pir, ) def initConfig(self): @@ -337,6 +344,8 @@ def initConfig(self): self.epsilon = 1e-05 self.data_format = "NCHW" self.use_global_stats = None + # Todo(CZ): open this + self.check_prim_pir = False @unittest.skipIf( diff --git a/test/legacy_test/test_batch_norm_op_prim_nhwc.py b/test/legacy_test/test_batch_norm_op_prim_nhwc.py index bfa104bcf73d40..57041857c00426 100644 --- a/test/legacy_test/test_batch_norm_op_prim_nhwc.py +++ b/test/legacy_test/test_batch_norm_op_prim_nhwc.py @@ -161,6 +161,8 @@ def initConfig(self): self.epsilon = 1e-05 self.data_format = "NHWC" self.use_global_stats = None + # Todo(CZ): open this + self.check_prim_pir = False class TestBatchNormOpNHWCShape2(TestBatchNormOp): From e134bc54e65b82d48300863f6de7745545954a52 Mon Sep 17 00:00:00 2001 From: Winters Montagne <118546135+WintersMontagne10335@users.noreply.github.com> Date: Mon, 20 Nov 2023 18:43:20 +0800 Subject: [PATCH 28/46] modified: paddle/phi/kernels/gpu/contiguous_kernel.cu (#58303) --- paddle/phi/kernels/gpu/contiguous_kernel.cu | 518 ++++++++++++-------- 1 file changed, 311 insertions(+), 207 deletions(-) diff --git a/paddle/phi/kernels/gpu/contiguous_kernel.cu b/paddle/phi/kernels/gpu/contiguous_kernel.cu index 49b253effd9451..ff53a9456182fb 100644 --- a/paddle/phi/kernels/gpu/contiguous_kernel.cu +++ b/paddle/phi/kernels/gpu/contiguous_kernel.cu @@ -20,6 +20,12 @@ limitations under the License. */ #include "paddle/phi/kernels/transpose_kernel.h" namespace phi { +bool VerifyThreadConfigurationParameters(const dim3& block, const dim3& grid) { + return block.x <= 1024 && block.y <= 1024 && block.z <= 64 && + block.x * block.y * block.z <= 1024 && + block.x * block.y * block.z >= 96 && grid.y < 65536 && grid.z < 65536; +} + template __global__ void ContiguousCaseZeroFunc( const T* input_data, @@ -137,6 +143,28 @@ __global__ void ContiguousCaseOneFunc( } } +template +__global__ void ContiguousDefaultFunc( + const T* input_data, + phi::Array input_stride, + phi::Array dims, + const int64_t numel, + T* out_data) { + int64_t gid = blockIdx.x * blockDim.x + threadIdx.x; +#pragma unroll + for (int64_t i = gid; i < numel; i += blockDim.x * gridDim.x) { + int64_t input_offset = 0; + int64_t index_tmp = i; +#pragma unroll + for (int dim = N - 1; dim >= 0; --dim) { + input_offset += index_tmp % dims[dim] * input_stride[dim]; + index_tmp = index_tmp / dims[dim]; + } + + out_data[i] = input_data[input_offset]; + } +} + bool is_only_transposed(const DDim& shape, const DDim& stride, uint64_t offset, @@ -182,6 +210,273 @@ bool is_only_transposed(const DDim& shape, } } +template +bool LaunchContiguousCazeZeroKernel( + const Context& dev_ctx, + const T* input_data, + const phi::Array& input_stride, + const phi::Array& input_dims, + int rank, + T* output_data) { + if (rank > 6) { + return false; + } + + dim3 grid(1, 1, 1), block(1, 1, 1); + + if (rank >= 1) { + block.x = input_dims[rank - 1]; + } + + if (rank >= 2) { + block.y = input_dims[rank - 2]; + } + + if (rank >= 3) { + block.z = input_dims[rank - 3]; + } + + if (rank >= 4) { + grid.x = input_dims[rank - 4]; + } + + if (rank >= 5) { + grid.y = input_dims[rank - 5]; + } + + if (rank >= 6) { + grid.z = input_dims[rank - 6]; + } + + if (!VerifyThreadConfigurationParameters(block, grid)) { + return false; + } + + switch (rank) { + case 1: + ContiguousCaseZeroFunc<<>>( + input_data, output_data, input_stride); + break; + case 2: + ContiguousCaseZeroFunc<<>>( + input_data, output_data, input_stride); + break; + case 3: + ContiguousCaseZeroFunc<<>>( + input_data, output_data, input_stride); + break; + case 4: + ContiguousCaseZeroFunc<<>>( + input_data, output_data, input_stride); + break; + case 5: + ContiguousCaseZeroFunc<<>>( + input_data, output_data, input_stride); + break; + case 6: + ContiguousCaseZeroFunc<<>>( + input_data, output_data, input_stride); + break; + } + + return true; +} + +template +bool LaunchContiguousCazeOneKernel( + const Context& dev_ctx, + const T* input_data, + const phi::Array& input_stride, + const phi::Array& input_dims, + int rank, + int numel, + T* output_data) { + dim3 grid(1, 1, 1), block(1, 1, 1); + phi::Array cur_input_dims; + block.x = 512; + + if (rank >= 1) { + grid.x = (numel + block.x - 1) / block.x; + cur_input_dims[0] = input_dims[rank - 1]; + } + + if (rank >= 2) { + cur_input_dims[1] = input_dims[rank - 2]; + } + + if (rank >= 4) { + grid.x = + (input_dims[rank - 1] * input_dims[rank - 2] * input_dims[rank - 3] + + block.x - 1) / + block.x; + grid.y = input_dims[rank - 4]; + cur_input_dims[2] = input_dims[rank - 4]; + } + + if (rank >= 5) { + grid.y = input_dims[rank - 4] * input_dims[rank - 5]; + cur_input_dims[2] = input_dims[rank - 4]; + cur_input_dims[3] = input_dims[rank - 5]; + } + + if (rank >= 6) { + grid.y = input_dims[rank - 4] * input_dims[rank - 5] * input_dims[rank - 6]; + } + + if (rank >= 7) { + grid.z = input_dims[rank - 7]; + cur_input_dims[4] = input_dims[rank - 7]; + } + + if (rank >= 8) { + grid.z = input_dims[rank - 7] * input_dims[rank - 8]; + cur_input_dims[5] = input_dims[rank - 8]; + } + + if (rank >= 9) { + grid.z = input_dims[rank - 7] * input_dims[rank - 8] * input_dims[rank - 9]; + } + + if (!VerifyThreadConfigurationParameters(block, grid)) { + return false; + } + + switch (rank) { + case 1: + ContiguousCaseOneFunc + <<>>(input_data, + output_data, + input_stride, + cur_input_dims, + input_dims[rank - 1]); + break; + case 2: + ContiguousCaseOneFunc<<>>( + input_data, + output_data, + input_stride, + cur_input_dims, + input_dims[rank - 1] * input_dims[rank - 2]); + break; + case 3: + ContiguousCaseOneFunc<<>>( + input_data, + output_data, + input_stride, + cur_input_dims, + input_dims[rank - 1] * input_dims[rank - 2] * input_dims[rank - 3]); + break; + case 4: + ContiguousCaseOneFunc<<>>( + input_data, + output_data, + input_stride, + cur_input_dims, + input_dims[rank - 1] * input_dims[rank - 2] * input_dims[rank - 3]); + break; + case 5: + ContiguousCaseOneFunc<<>>( + input_data, + output_data, + input_stride, + cur_input_dims, + input_dims[rank - 1] * input_dims[rank - 2] * input_dims[rank - 3]); + break; + case 6: + ContiguousCaseOneFunc<<>>( + input_data, + output_data, + input_stride, + cur_input_dims, + input_dims[rank - 1] * input_dims[rank - 2] * input_dims[rank - 3]); + break; + case 7: + ContiguousCaseOneFunc<<>>( + input_data, + output_data, + input_stride, + cur_input_dims, + input_dims[rank - 1] * input_dims[rank - 2] * input_dims[rank - 3]); + break; + case 8: + ContiguousCaseOneFunc<<>>( + input_data, + output_data, + input_stride, + cur_input_dims, + input_dims[rank - 1] * input_dims[rank - 2] * input_dims[rank - 3]); + break; + case 9: + ContiguousCaseOneFunc<<>>( + input_data, + output_data, + input_stride, + cur_input_dims, + input_dims[rank - 1] * input_dims[rank - 2] * input_dims[rank - 3]); + break; + default: + PADDLE_THROW(phi::errors::InvalidArgument( + "The rank of input should be less than 9, but received %d.", rank)); + } + + return true; +} + +template +void LaunchContiguousDefaultKernel( + const Context& dev_ctx, + const T* input_data, + const phi::Array& input_stride, + const phi::Array& input_dims, + int rank, + int numel, + T* output_data) { + int64_t block = 512; + int64_t grid = (numel + block - 1) / block; + + switch (rank) { + case 1: + ContiguousDefaultFunc<<>>( + input_data, input_stride, input_dims, numel, output_data); + break; + case 2: + ContiguousDefaultFunc<<>>( + input_data, input_stride, input_dims, numel, output_data); + break; + case 3: + ContiguousDefaultFunc<<>>( + input_data, input_stride, input_dims, numel, output_data); + break; + case 4: + ContiguousDefaultFunc<<>>( + input_data, input_stride, input_dims, numel, output_data); + break; + case 5: + ContiguousDefaultFunc<<>>( + input_data, input_stride, input_dims, numel, output_data); + break; + case 6: + ContiguousDefaultFunc<<>>( + input_data, input_stride, input_dims, numel, output_data); + break; + case 7: + ContiguousDefaultFunc<<>>( + input_data, input_stride, input_dims, numel, output_data); + break; + case 8: + ContiguousDefaultFunc<<>>( + input_data, input_stride, input_dims, numel, output_data); + break; + case 9: + ContiguousDefaultFunc<<>>( + input_data, input_stride, input_dims, numel, output_data); + break; + default: + PADDLE_THROW(phi::errors::InvalidArgument( + "The rank of input should be less than 9, but received %d.", rank)); + } +} + template void ContiguousKernel(const Context& dev_ctx, const DenseTensor& input, @@ -229,214 +524,23 @@ void ContiguousKernel(const Context& dev_ctx, input_stride[0] = 1; } - dim3 grid(1, 1, 1), block(1, 1, 1); - - int tmp = 1; - - for (int i = 0; i < 3 && i < rank; i++) { - tmp *= input_dims[rank - 1 - i]; - } - - if (rank <= 6 && tmp <= 1024 && - (input_dims.size() < 3 || input_dims[rank - 3] <= 64)) { - if (rank >= 1) { - block.x = input_dims[rank - 1]; - } - - if (rank >= 2) { - block.y = input_dims[rank - 2]; - } - - if (rank >= 3) { - block.z = input_dims[rank - 3]; - } - - switch (rank) { - case 1: - ContiguousCaseZeroFunc<<>>( - input_data, output_data, input_stride); - break; - case 2: - ContiguousCaseZeroFunc<<>>( - input_data, output_data, input_stride); - break; - case 3: - ContiguousCaseZeroFunc<<>>( - input_data, output_data, input_stride); - break; - case 4: - grid.x = input_dims[rank - 4]; - ContiguousCaseZeroFunc<<>>( - input_data, output_data, input_stride); - break; - case 5: - grid.x = input_dims[rank - 4]; - grid.y = input_dims[rank - 5]; - ContiguousCaseZeroFunc<<>>( - input_data, output_data, input_stride); - break; - case 6: - grid.x = input_dims[rank - 4]; - grid.y = input_dims[rank - 5]; - grid.z = input_dims[rank - 6]; - ContiguousCaseZeroFunc<<>>( - input_data, output_data, input_stride); - break; - } + if (LaunchContiguousCazeZeroKernel( + dev_ctx, input_data, input_stride, input_dims, rank, output_data)) { + } else if (LaunchContiguousCazeOneKernel(dev_ctx, + input_data, + input_stride, + input_dims, + rank, + numel, + output_data)) { } else { - phi::Array cur_input_dims; - block.x = 512; - switch (rank) { - case 1: - grid.x = (numel + block.x - 1) / block.x; - cur_input_dims[0] = input_dims[rank - 1]; - ContiguousCaseOneFunc - <<>>(input_data, - output_data, - input_stride, - cur_input_dims, - input_dims[rank - 1]); - break; - case 2: - grid.x = (numel + block.x - 1) / block.x; - cur_input_dims[0] = input_dims[rank - 1]; - cur_input_dims[1] = input_dims[rank - 2]; - ContiguousCaseOneFunc<<>>( - input_data, - output_data, - input_stride, - cur_input_dims, - input_dims[rank - 1] * input_dims[rank - 2]); - break; - case 3: - grid.x = (numel + block.x - 1) / block.x; - cur_input_dims[0] = input_dims[rank - 1]; - cur_input_dims[1] = input_dims[rank - 2]; - ContiguousCaseOneFunc<<>>( - input_data, - output_data, - input_stride, - cur_input_dims, - input_dims[rank - 1] * input_dims[rank - 2] * input_dims[rank - 3]); - break; - case 4: - grid.x = (input_dims[rank - 1] * input_dims[rank - 2] * - input_dims[rank - 3] + - block.x - 1) / - block.x; - grid.y = input_dims[rank - 4]; - cur_input_dims[0] = input_dims[rank - 1]; - cur_input_dims[1] = input_dims[rank - 2]; - cur_input_dims[2] = input_dims[rank - 4]; - ContiguousCaseOneFunc<<>>( - input_data, - output_data, - input_stride, - cur_input_dims, - input_dims[rank - 1] * input_dims[rank - 2] * input_dims[rank - 3]); - break; - case 5: - grid.x = (input_dims[rank - 1] * input_dims[rank - 2] * - input_dims[rank - 3] + - block.x - 1) / - block.x; - grid.y = input_dims[rank - 4] * input_dims[rank - 5]; - cur_input_dims[0] = input_dims[rank - 1]; - cur_input_dims[1] = input_dims[rank - 2]; - cur_input_dims[2] = input_dims[rank - 4]; - cur_input_dims[3] = input_dims[rank - 5]; - ContiguousCaseOneFunc<<>>( - input_data, - output_data, - input_stride, - cur_input_dims, - input_dims[rank - 1] * input_dims[rank - 2] * input_dims[rank - 3]); - break; - case 6: - grid.x = (input_dims[rank - 1] * input_dims[rank - 2] * - input_dims[rank - 3] + - block.x - 1) / - block.x; - grid.y = - input_dims[rank - 4] * input_dims[rank - 5] * input_dims[rank - 6]; - cur_input_dims[0] = input_dims[rank - 1]; - cur_input_dims[1] = input_dims[rank - 2]; - cur_input_dims[2] = input_dims[rank - 4]; - cur_input_dims[3] = input_dims[rank - 5]; - ContiguousCaseOneFunc<<>>( - input_data, - output_data, - input_stride, - cur_input_dims, - input_dims[rank - 1] * input_dims[rank - 2] * input_dims[rank - 3]); - break; - case 7: - grid.x = (input_dims[rank - 1] * input_dims[rank - 2] * - input_dims[rank - 3] + - block.x - 1) / - block.x; - grid.y = - input_dims[rank - 4] * input_dims[rank - 5] * input_dims[rank - 6]; - grid.z = input_dims[rank - 7]; - cur_input_dims[0] = input_dims[rank - 1]; - cur_input_dims[1] = input_dims[rank - 2]; - cur_input_dims[2] = input_dims[rank - 4]; - cur_input_dims[3] = input_dims[rank - 5]; - cur_input_dims[4] = input_dims[rank - 7]; - ContiguousCaseOneFunc<<>>( - input_data, - output_data, - input_stride, - cur_input_dims, - input_dims[rank - 1] * input_dims[rank - 2] * input_dims[rank - 3]); - break; - case 8: - grid.x = (input_dims[rank - 1] * input_dims[rank - 2] * - input_dims[rank - 3] + - block.x - 1) / - block.x; - grid.y = - input_dims[rank - 4] * input_dims[rank - 5] * input_dims[rank - 6]; - grid.z = input_dims[rank - 7] * input_dims[rank - 8]; - cur_input_dims[0] = input_dims[rank - 1]; - cur_input_dims[1] = input_dims[rank - 2]; - cur_input_dims[2] = input_dims[rank - 4]; - cur_input_dims[3] = input_dims[rank - 5]; - cur_input_dims[4] = input_dims[rank - 7]; - cur_input_dims[5] = input_dims[rank - 8]; - ContiguousCaseOneFunc<<>>( - input_data, - output_data, - input_stride, - cur_input_dims, - input_dims[rank - 1] * input_dims[rank - 2] * input_dims[rank - 3]); - break; - case 9: - grid.x = (input_dims[rank - 1] * input_dims[rank - 2] * - input_dims[rank - 3] + - block.x - 1) / - block.x; - grid.y = - input_dims[rank - 4] * input_dims[rank - 5] * input_dims[rank - 6]; - grid.z = - input_dims[rank - 7] * input_dims[rank - 8] * input_dims[rank - 9]; - cur_input_dims[0] = input_dims[rank - 1]; - cur_input_dims[1] = input_dims[rank - 2]; - cur_input_dims[2] = input_dims[rank - 4]; - cur_input_dims[3] = input_dims[rank - 5]; - cur_input_dims[4] = input_dims[rank - 7]; - cur_input_dims[5] = input_dims[rank - 8]; - ContiguousCaseOneFunc<<>>( - input_data, - output_data, - input_stride, - cur_input_dims, - input_dims[rank - 1] * input_dims[rank - 2] * input_dims[rank - 3]); - break; - default: - PADDLE_THROW(phi::errors::InvalidArgument( - "The rank of input should be less than 9, but received %d.", rank)); - } + LaunchContiguousDefaultKernel(dev_ctx, + input_data, + input_stride, + input_dims, + rank, + numel, + output_data); } } From 8321febbd6a380c87e5db1a6fc514a9b4f58b9cf Mon Sep 17 00:00:00 2001 From: kevin Date: Mon, 20 Nov 2023 18:56:24 +0800 Subject: [PATCH 29/46] [Prim][PIR] sqrt forward sink and fix Decomp bug (#59135) * sqrt prim sink c++ * prim sqrt sink c++ * remove sqrt op with python --- .../decomp_interface_gen_op_list.py | 2 ++ .../fluid/pir/dialect/op_generator/op_gen.py | 34 +++++++++++-------- paddle/fluid/primitive/composite/composite.h | 22 ++++++++++++ python/paddle/decomposition/rules.py | 19 ----------- 4 files changed, 44 insertions(+), 33 deletions(-) diff --git a/paddle/fluid/pir/dialect/op_generator/decomp_interface_gen_op_list.py b/paddle/fluid/pir/dialect/op_generator/decomp_interface_gen_op_list.py index a59855dcb265cd..99903c1949febc 100644 --- a/paddle/fluid/pir/dialect/op_generator/decomp_interface_gen_op_list.py +++ b/paddle/fluid/pir/dialect/op_generator/decomp_interface_gen_op_list.py @@ -27,6 +27,7 @@ "softmax", "layer_norm", "gelu", + "sqrt", ] # come into effect in generated file op_decomp.cc @@ -39,6 +40,7 @@ "softmax", "layer_norm", "gelu", + "sqrt", ] diff --git a/paddle/fluid/pir/dialect/op_generator/op_gen.py b/paddle/fluid/pir/dialect/op_generator/op_gen.py index 27073c668fdfec..856f42ad1a8456 100644 --- a/paddle/fluid/pir/dialect/op_generator/op_gen.py +++ b/paddle/fluid/pir/dialect/op_generator/op_gen.py @@ -1185,22 +1185,9 @@ def OpGenerator( # If op has inplace info, we will generate inplace op and non-inplace op. for op_name in op_info.op_phi_name: - if op_name in decomp_interface_declare_gen_op_list: - op_interfaces = op_interfaces + [ - "paddle::dialect::DecompInterface" - ] - exclusive_interface_str += "\n static std::vector> Decomp(pir::Operation* op);" - else: - op_interfaces = op_interfaces_tmp - exclusive_interface_str = exclusive_interface_str_tmp - # =================================== # - # gen interface/trait list str # + # gen trait list str # # =================================== # - op_interfaces_str = "" - if len(op_interfaces) > 0: - op_interfaces_str = "," + ",".join(op_interfaces) - if op_name[-1] == "_": op_traits += ["paddle::dialect::InplaceTrait"] @@ -1216,6 +1203,25 @@ def OpGenerator( func_list = op_kernel_map['func'] for kernel_func_name in func_list: + if ( + op_name in decomp_interface_declare_gen_op_list + and kernel_func_name in decomp_interface_declare_gen_op_list + ): + op_interfaces = op_interfaces + [ + "paddle::dialect::DecompInterface" + ] + exclusive_interface_str += "\n static std::vector> Decomp(pir::Operation* op);" + else: + op_interfaces = op_interfaces_tmp + exclusive_interface_str = exclusive_interface_str_tmp + + # =================================== # + # gen interface list str # + # =================================== # + op_interfaces_str = "" + if len(op_interfaces) > 0: + op_interfaces_str = "," + ",".join(op_interfaces) + if len(func_list) == 1: op_class_name = to_pascal_case(op_name) + "Op" op_dialect_name = dialect_name + "." + op_name diff --git a/paddle/fluid/primitive/composite/composite.h b/paddle/fluid/primitive/composite/composite.h index 572834437a4b5f..4e20773b3dbc49 100644 --- a/paddle/fluid/primitive/composite/composite.h +++ b/paddle/fluid/primitive/composite/composite.h @@ -304,6 +304,28 @@ std::tuple layer_norm_decomp( return std::make_tuple(out, mean_, variance); } +template +Tensor sqrt_decomp(const Tensor& x) { + auto org_dtype = x.dtype(); + bool need_cast = + org_dtype == phi::DataType::FLOAT16 || org_dtype == phi::DataType::UINT16; + + Tensor x_cast; + if (need_cast) { + x_cast = cast(x, phi::DataType::FLOAT32); + } else { + x_cast = x; + } + + auto ans = elementwise_pow( + x_cast, full(phi::vectorize(x_cast.dims()), 0.5, x_cast.dtype())); + if (need_cast) { + return cast(ans, org_dtype); + } else { + return ans; + } +} + template Tensor gelu_decomp(const Tensor& x, bool approximate) { const double PM_2_SQRTPI = 1.12837916709551257390; /* 2/sqrt(pi) */ diff --git a/python/paddle/decomposition/rules.py b/python/paddle/decomposition/rules.py index bd8a58fc680a36..d9972846a63317 100644 --- a/python/paddle/decomposition/rules.py +++ b/python/paddle/decomposition/rules.py @@ -36,25 +36,6 @@ def mean(x, axis, keepdim): return res -@register_decomp('pd_op.sqrt') -def sqrt(x): - """ - define composite rule of op sqrt - res = pow(x, 0.5) - """ - is_amp = False - from paddle.base.data_feeder import convert_dtype - - dtype = convert_dtype(x.dtype) - if dtype in ["float16", "uint16"]: - is_amp = True - x = cast(x, "float32") - - y = full(x.shape if len(x.shape) == 0 else [1], 0.5, x.dtype) - res = pow_composite(x, y) - return res if not is_amp else cast(res, dtype) - - @register_decomp('pd_op.rsqrt') def rsqrt(x): """define composite rule of op rsqrt.""" From cfdc8fa81e459415328dbd2faa51ebf749d9b4f2 Mon Sep 17 00:00:00 2001 From: WangZhen <23097963+0x45f@users.noreply.github.com> Date: Mon, 20 Nov 2023 19:07:06 +0800 Subject: [PATCH 30/46] [PIR]Gen check DataType (#58954) --- paddle/fluid/pir/dialect/CMakeLists.txt | 2 +- .../fluid/pir/dialect/op_generator/api_gen.py | 34 ++++++++ .../fluid/pir/dialect/operator/utils/utils.cc | 79 ++++++++++++++++++- .../fluid/pir/dialect/operator/utils/utils.h | 8 ++ paddle/phi/kernels/cpu/full_kernel.cc | 1 + test/legacy_test/test_allclose_op.py | 18 +++-- test/legacy_test/test_clip_op.py | 28 ++++--- test/legacy_test/test_cumsum_op.py | 22 +++--- test/legacy_test/test_isclose_op.py | 23 +++--- 9 files changed, 175 insertions(+), 40 deletions(-) diff --git a/paddle/fluid/pir/dialect/CMakeLists.txt b/paddle/fluid/pir/dialect/CMakeLists.txt index f567c506abab76..068fe1d5fa6d78 100644 --- a/paddle/fluid/pir/dialect/CMakeLists.txt +++ b/paddle/fluid/pir/dialect/CMakeLists.txt @@ -202,7 +202,7 @@ list( set(op_dialect_srcs ${op_dialect_srcs} ${op_source_file} ${api_source_file}) -set(op_dialect_deps phi pir type_info) +set(op_dialect_deps phi pir type_info string_helper) cc_library( op_dialect diff --git a/paddle/fluid/pir/dialect/op_generator/api_gen.py b/paddle/fluid/pir/dialect/op_generator/api_gen.py index 56e7b85ab705e2..355aa79a48a898 100644 --- a/paddle/fluid/pir/dialect/op_generator/api_gen.py +++ b/paddle/fluid/pir/dialect/op_generator/api_gen.py @@ -71,6 +71,7 @@ API_IMPL_TEMPLATE = """ {ret_type} {api_name}({args}){{ + {check_data_type} {handle_optional_inputs} {in_combine} {compute_op} @@ -81,6 +82,9 @@ """ +CHECK_DATA_TYPE_TEMPLATE = """ + {function}({input}, "{input}", "{op_name}");""" + OPTIONAL_VECTOR_VALUE_INPUT_TEMPLATE = """ paddle::optional optional_{name}; if (!{name}) {{ @@ -517,6 +521,35 @@ def _gen_return_result(self, ret_list): elif len(ret_list) == 0: return 'return;' + def _gen_check_data_type(self, op_info, op_name): + name_list = op_info.input_name_list + type_list = op_info.input_type_list + if ( + op_name.endswith(('_grad', '_grad_', '_grad_dense', '_grad_sparse')) + or len(name_list) == 0 + ): + return '' + try: + data_type_candidates = op_info.kernel_map['data_type']['candidates'] + except Exception: + data_type_candidates = None + ret = '' + if data_type_candidates is not None: + for name in data_type_candidates: + if name not in name_list: + continue + index = name_list.index(name) + type = type_list[index] + if VECTOR_TYPE in type: + function_name = 'CheckVectorOfValueDataType' + + else: + function_name = 'CheckValueDataType' + ret += CHECK_DATA_TYPE_TEMPLATE.format( + function=function_name, input=name, op_name=op_name + ) + return ret + def _gen_one_impl( self, op_info, op_name, is_mutable_attr, is_vector_mutable_attr ): @@ -535,6 +568,7 @@ def _gen_one_impl( ) ret = API_IMPL_TEMPLATE.format( + check_data_type=self._gen_check_data_type(op_info, op_name), ret_type=ret_type, api_name=op_name, args=self._gen_api_args( diff --git a/paddle/fluid/pir/dialect/operator/utils/utils.cc b/paddle/fluid/pir/dialect/operator/utils/utils.cc index 56ee9d11f3fe80..8a5501c5c7a17e 100644 --- a/paddle/fluid/pir/dialect/operator/utils/utils.cc +++ b/paddle/fluid/pir/dialect/operator/utils/utils.cc @@ -12,11 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "paddle/fluid/pir/dialect/operator/utils/utils.h" +#include +#include #include + +#include "paddle/fluid/framework/phi_utils.h" #include "paddle/fluid/pir/dialect/operator/ir/manual_op.h" #include "paddle/fluid/pir/dialect/operator/ir/op_attribute.h" #include "paddle/fluid/pir/dialect/operator/ir/pd_op.h" +#include "paddle/fluid/pir/dialect/operator/utils/utils.h" +#include "paddle/phi/core/kernel_factory.h" +#include "paddle/pir/core/builtin_type.h" +#include "paddle/utils/string/string_helper.h" namespace paddle { namespace dialect { @@ -227,5 +234,75 @@ std::vector GetInt64Vector(const pir::Attribute& attr) { return vec_int64; } +std::set GetRegisterDataType(const std::string& op_name) { + std::string non_inplace_op_name; + if (paddle::string::ends_with(op_name, "_")) { + non_inplace_op_name = op_name.substr(0, op_name.size() - 1); + } + + std::set data_type; + auto& phi_kernels = phi::KernelFactory::Instance().kernels(); + for (auto& kernel_pair : phi_kernels) { + auto fluid_op_name = phi::TransToFluidOpName(kernel_pair.first); + if (kernel_pair.first != op_name && fluid_op_name != op_name && + kernel_pair.first != non_inplace_op_name && + fluid_op_name != non_inplace_op_name) { + continue; + } + for (auto& info_pair : kernel_pair.second) { + data_type.insert(phi::DataTypeToString(info_pair.first.dtype())); + } + } + + return data_type; +} + +void DoCheck(const pir::Value& value, + const std::string& input_name, + const std::set& expected_dtype, + const std::string& op_name) { + if (value.type().isa()) { + std::string value_type = phi::DataTypeToString(dialect::TransToPhiDataType( + value.type().dyn_cast().dtype())); + if (expected_dtype.find(value_type) == expected_dtype.end()) { + std::ostringstream joined; + std::copy(expected_dtype.begin(), + expected_dtype.end(), + std::ostream_iterator(joined, ",")); + PADDLE_THROW(phi::errors::InvalidArgument( + "Check data type error for op: %s, input: %s, %s.dtype is %s, and " + "expected_dtype is %s", + op_name, + input_name, + input_name, + value_type, + joined.str())); + } + } else { + PADDLE_THROW(phi::errors::InvalidArgument( + "Currently, we can only get dtype for dense " + "tensor.")); + } +} + +void CheckValueDataType(const pir::Value& value, + const std::string& input_name, + const std::string& op_name) { + VLOG(6) << "CheckValueDataType for " << op_name << ", input: " << input_name; + std::set expected_dtype = GetRegisterDataType(op_name); + DoCheck(value, input_name, expected_dtype, op_name); +} + +void CheckVectorOfValueDataType(const std::vector& vector_value, + const std::string& input_name, + const std::string& op_name) { + VLOG(6) << "CheckVectorOfValueDataType for " << op_name + << ", input: " << input_name; + std::set expected_dtype = GetRegisterDataType(op_name); + for (auto& value : vector_value) { + DoCheck(value, input_name, expected_dtype, op_name); + } +} + } // namespace dialect } // namespace paddle diff --git a/paddle/fluid/pir/dialect/operator/utils/utils.h b/paddle/fluid/pir/dialect/operator/utils/utils.h index e35d7fa74cc649..4bbd454d3ea350 100644 --- a/paddle/fluid/pir/dialect/operator/utils/utils.h +++ b/paddle/fluid/pir/dialect/operator/utils/utils.h @@ -135,5 +135,13 @@ bool IsEmptyValue(const pir::Value& value); std::vector GetInt64Vector(const pir::Attribute& attr); +void CheckValueDataType(const pir::Value& value, + const std::string& input_name, + const std::string& op_name); + +void CheckVectorOfValueDataType(const std::vector& vector_value, + const std::string& input_name, + const std::string& op_name); + } // namespace dialect } // namespace paddle diff --git a/paddle/phi/kernels/cpu/full_kernel.cc b/paddle/phi/kernels/cpu/full_kernel.cc index bb2533490cfc29..e4ba06778817c0 100644 --- a/paddle/phi/kernels/cpu/full_kernel.cc +++ b/paddle/phi/kernels/cpu/full_kernel.cc @@ -124,6 +124,7 @@ PD_REGISTER_KERNEL(full_like, phi::FullLikeKernel, float, double, + uint8_t, int16_t, int, int64_t, diff --git a/test/legacy_test/test_allclose_op.py b/test/legacy_test/test_allclose_op.py index cb76671284e2c0..68a47364c897d3 100644 --- a/test/legacy_test/test_allclose_op.py +++ b/test/legacy_test/test_allclose_op.py @@ -177,13 +177,17 @@ def test_equal_nan(): class TestAllcloseOpFp16(unittest.TestCase): @test_with_pir_api def test_fp16(self): - x_data = np.random.rand(10, 10).astype('float16') - y_data = np.random.rand(10, 10).astype('float16') - with paddle.static.program_guard(paddle.static.Program()): - x = paddle.static.data(shape=[10, 10], name='x', dtype='float16') - y = paddle.static.data(shape=[10, 10], name='y', dtype='float16') - out = paddle.allclose(x, y, rtol=1e-05, atol=1e-08) - if core.is_compiled_with_cuda(): + if core.is_compiled_with_cuda(): + x_data = np.random.rand(10, 10).astype('float16') + y_data = np.random.rand(10, 10).astype('float16') + with paddle.static.program_guard(paddle.static.Program()): + x = paddle.static.data( + shape=[10, 10], name='x', dtype='float16' + ) + y = paddle.static.data( + shape=[10, 10], name='y', dtype='float16' + ) + out = paddle.allclose(x, y, rtol=1e-05, atol=1e-08) place = paddle.CUDAPlace(0) exe = paddle.static.Executor(place) exe.run(paddle.static.default_startup_program()) diff --git a/test/legacy_test/test_clip_op.py b/test/legacy_test/test_clip_op.py index 1fad87de2d1dce..64b93aa96eaf84 100644 --- a/test/legacy_test/test_clip_op.py +++ b/test/legacy_test/test_clip_op.py @@ -436,18 +436,22 @@ def test_errors(self): class TestClipOpFp16(unittest.TestCase): @test_with_pir_api def test_fp16(self): - paddle.enable_static() - data_shape = [1, 9, 9, 4] - data = np.random.random(data_shape).astype('float16') + if base.core.is_compiled_with_cuda(): + paddle.enable_static() + data_shape = [1, 9, 9, 4] + data = np.random.random(data_shape).astype('float16') - with paddle.static.program_guard(paddle.static.Program()): - images = paddle.static.data( - name='image1', shape=data_shape, dtype='float16' - ) - min = paddle.static.data(name='min1', shape=[1], dtype='float16') - max = paddle.static.data(name='max1', shape=[1], dtype='float16') - out = paddle.clip(images, min, max) - if base.core.is_compiled_with_cuda(): + with paddle.static.program_guard(paddle.static.Program()): + images = paddle.static.data( + name='image1', shape=data_shape, dtype='float16' + ) + min = paddle.static.data( + name='min1', shape=[1], dtype='float16' + ) + max = paddle.static.data( + name='max1', shape=[1], dtype='float16' + ) + out = paddle.clip(images, min, max) place = paddle.CUDAPlace(0) exe = paddle.static.Executor(place) res1 = exe.run( @@ -458,7 +462,7 @@ def test_fp16(self): }, fetch_list=[out], ) - paddle.disable_static() + paddle.disable_static() class TestInplaceClipAPI(TestClipAPI): diff --git a/test/legacy_test/test_cumsum_op.py b/test/legacy_test/test_cumsum_op.py index ee853bd553eb0d..e17fcd2abf3e67 100644 --- a/test/legacy_test/test_cumsum_op.py +++ b/test/legacy_test/test_cumsum_op.py @@ -562,20 +562,22 @@ def test_static_and_infer(self): class TestCumSumOpFp16(unittest.TestCase): @test_with_pir_api def test_fp16(self): - paddle.enable_static() - x_np = np.random.random((100, 100)).astype('float16') - with paddle.static.program_guard(paddle.static.Program()): - x = paddle.static.data(shape=[100, 100], name='x', dtype='float16') - y1 = paddle.cumsum(x) - y2 = paddle.cumsum(x, axis=0) - y3 = paddle.cumsum(x, axis=-1) - y4 = paddle.cumsum(x, axis=-2) - if core.is_compiled_with_cuda(): + if core.is_compiled_with_cuda(): + paddle.enable_static() + x_np = np.random.random((100, 100)).astype('float16') + with paddle.static.program_guard(paddle.static.Program()): + x = paddle.static.data( + shape=[100, 100], name='x', dtype='float16' + ) + y1 = paddle.cumsum(x) + y2 = paddle.cumsum(x, axis=0) + y3 = paddle.cumsum(x, axis=-1) + y4 = paddle.cumsum(x, axis=-2) place = paddle.CUDAPlace(0) exe = paddle.static.Executor(place) exe.run(paddle.static.default_startup_program()) out = exe.run(feed={'x': x_np}, fetch_list=[y1, y2, y3, y4]) - paddle.disable_static() + paddle.disable_static() if __name__ == '__main__': diff --git a/test/legacy_test/test_isclose_op.py b/test/legacy_test/test_isclose_op.py index c9803bf4411492..414669ae163066 100644 --- a/test/legacy_test/test_isclose_op.py +++ b/test/legacy_test/test_isclose_op.py @@ -213,15 +213,20 @@ def test_equal_nan(): class TestIscloseOpFp16(unittest.TestCase): @test_with_pir_api def test_fp16(self): - x_data = np.random.rand(10, 10).astype('float16') - y_data = np.random.rand(10, 10).astype('float16') - main = paddle.static.Program() - startup = paddle.static.Program() - with paddle.static.program_guard(main, startup): - x = paddle.static.data(shape=[10, 10], name='x', dtype='float16') - y = paddle.static.data(shape=[10, 10], name='y', dtype='float16') - out = paddle.isclose(x, y, rtol=1e-05, atol=1e-08) - if core.is_compiled_with_cuda(): + if core.is_compiled_with_cuda(): + x_data = np.random.rand(10, 10).astype('float16') + y_data = np.random.rand(10, 10).astype('float16') + main = paddle.static.Program() + startup = paddle.static.Program() + with paddle.static.program_guard(main, startup): + x = paddle.static.data( + shape=[10, 10], name='x', dtype='float16' + ) + y = paddle.static.data( + shape=[10, 10], name='y', dtype='float16' + ) + out = paddle.isclose(x, y, rtol=1e-05, atol=1e-08) + place = paddle.CUDAPlace(0) exe = paddle.static.Executor(place) exe.run(startup) From 471ffef5eab777ee10ba65f3e71aecbfac6672a3 Mon Sep 17 00:00:00 2001 From: wanghuancoder Date: Mon, 20 Nov 2023 19:11:27 +0800 Subject: [PATCH 31/46] [AutoParallel] GetTensorFromArgs check convert disttensor (#59139) * GetTensorFromArgs_check_convert_disttensor --- paddle/fluid/pybind/eager_functions.cc | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/paddle/fluid/pybind/eager_functions.cc b/paddle/fluid/pybind/eager_functions.cc index 6175e0fcae972a..441ffa1041f3ea 100644 --- a/paddle/fluid/pybind/eager_functions.cc +++ b/paddle/fluid/pybind/eager_functions.cc @@ -937,6 +937,11 @@ static PyObject* eager_api_async_read(PyObject* self, auto& buffer = GetTensorFromArgs("async_read", "buffer", args, 3, false); auto& offset = GetTensorFromArgs("async_read", "offset", args, 4, false); auto& count = GetTensorFromArgs("async_read", "count", args, 5, false); + const phi::distributed::ProcessMesh* mesh = nullptr; + if (InputsContainDistTensor(&mesh, src, dst, index, buffer, offset, count)) { + ConvertAllInputsToDistTensor(mesh, src, dst, index, buffer, offset, count); + } + { eager_gil_scoped_release guard; PADDLE_ENFORCE_EQ( @@ -1111,6 +1116,10 @@ static PyObject* eager_api_async_write(PyObject* self, auto& dst = GetTensorFromArgs("async_write", "dst", args, 1, false); auto& offset = GetTensorFromArgs("async_write", "offset", args, 2, false); auto& count = GetTensorFromArgs("async_write", "count", args, 3, false); + const phi::distributed::ProcessMesh* mesh = nullptr; + if (InputsContainDistTensor(&mesh, src, dst, offset, count)) { + ConvertAllInputsToDistTensor(mesh, src, dst, offset, count); + } { eager_gil_scoped_release guard; PADDLE_ENFORCE_EQ( From cf4cf130ec7b343c330df72fc76b99abf00f9d1c Mon Sep 17 00:00:00 2001 From: zhangbo9674 <82555433+zhangbo9674@users.noreply.github.com> Date: Mon, 20 Nov 2023 19:21:50 +0800 Subject: [PATCH 32/46] fix (#59147) --- paddle/phi/api/yaml/op_compat.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/paddle/phi/api/yaml/op_compat.yaml b/paddle/phi/api/yaml/op_compat.yaml index e40398f7773915..cb8c787abd8d33 100755 --- a/paddle/phi/api/yaml/op_compat.yaml +++ b/paddle/phi/api/yaml/op_compat.yaml @@ -3297,6 +3297,12 @@ outputs : out: Out +- op: c_reducescatter + inputs : + x : X + outputs : + out: Out + - op: c_sync_calc_stream inputs : x : X From d60251368b9cfb01275be8922a6e8081e24ecb3d Mon Sep 17 00:00:00 2001 From: hong <43953930+phlrain@users.noreply.github.com> Date: Mon, 20 Nov 2023 19:45:24 +0800 Subject: [PATCH 33/46] [PIR]Test llama sub graph ROPE (#59087) * fix layer norm fusion merge bug * update * [PIR+CINN]Add some subgraph from Llama for CINN * update * update * remove usless code * pir cinn support multi group * update * update * test llama sub graph ROPE * update * update * update * fix bug --------- Co-authored-by: Aurelius84 --- .../hlir/dialect/operator/ir/manual_op.cc | 51 +++++++ .../cinn/hlir/dialect/operator/ir/manual_op.h | 19 +++ .../hlir/dialect/operator/ir/op_dialect.cc | 1 + paddle/cinn/hlir/dialect/operator/ir/ops.yaml | 8 ++ .../add_broadcast_to_elementwise_pass.cc | 3 +- .../transforms/cinn_group_lowering_pass.cc | 5 +- .../transforms/op_with_group_merge_pass.cc | 4 + .../operator/transforms/pd_to_cinn_pass.cc | 79 ++++++++++- paddle/cinn/hlir/framework/pir/op_mapper.cc | 66 ++++++--- paddle/cinn/hlir/framework/pir/utils.cc | 1 + paddle/cinn/hlir/framework/pir_compiler.cc | 2 +- .../transforms/dead_code_elimination_pass.cc | 4 + paddle/fluid/pybind/pir.cc | 2 + python/paddle/base/variable_index.py | 14 +- test/cpp/pir/cinn/pir_all_path_test.cc | 134 ++++++++++++++++++ test/ir/pir/cinn/test_cinn_sub_graph.py | 1 - test/ir/pir/cinn/test_llama_sub_graph.py | 19 +-- 17 files changed, 369 insertions(+), 44 deletions(-) diff --git a/paddle/cinn/hlir/dialect/operator/ir/manual_op.cc b/paddle/cinn/hlir/dialect/operator/ir/manual_op.cc index 3a7341602a7641..6329a361be47d2 100644 --- a/paddle/cinn/hlir/dialect/operator/ir/manual_op.cc +++ b/paddle/cinn/hlir/dialect/operator/ir/manual_op.cc @@ -16,6 +16,7 @@ #include #include "glog/logging.h" +#include "paddle/fluid/pir/dialect/operator/ir/op_type.h" #include "paddle/pir/core/builtin_type.h" #include "paddle/pir/core/enforce.h" #include "paddle/pir/core/op_base.h" @@ -25,6 +26,7 @@ namespace cinn { namespace dialect { const char *GroupOp::attributes_name[GroupOp::attributes_num] = {"group_info"}; +const char *ConcatOp::attributes_name[GroupOp::attributes_num] = {"axis"}; void GroupOp::Build(pir::Builder &builder, pir::OperationArgument &argument, @@ -79,7 +81,56 @@ void GroupOp::Print(pir::IrPrinter &printer) { os << " \n }"; } +void ConcatOp::Build(pir::Builder &builder, // NOLINT + pir::OperationArgument &argument, // NOLINT + const std::vector &inputs, + int axis) { + VLOG(4) << "Start build ConcatOp"; + + argument.inputs = inputs; + std::vector inputs_type(inputs.size()); + + IR_ENFORCE(inputs.size() > 0); + + auto first_ele = + inputs[0].type().dyn_cast(); + phi::DDim out_dims = first_ele.dims(); + + if (axis < 0) { + axis += out_dims.size(); + } + + for (size_t idx = 0; idx < inputs.size(); ++idx) { + inputs_type[idx] = inputs[idx].type(); + + if (idx > 0) { + auto dim_i = inputs[idx] + .type() + .dyn_cast() + .dims(); + + out_dims[axis] += dim_i[axis]; + } + } + + auto out_type = + paddle::dialect::DenseTensorType::get(pir::IrContext::Instance(), + first_ele.dtype(), + out_dims, + first_ele.data_layout(), + first_ele.lod(), + first_ele.offset()); + + argument.output_types.emplace_back(out_type); + + PassStopGradientsDefaultly(argument); + + argument.AddAttribute( + "axis", pir::Int32Attribute::get(pir::IrContext::Instance(), axis)); +} + } // namespace dialect } // namespace cinn IR_DEFINE_EXPLICIT_TYPE_ID(cinn::dialect::GroupOp) +IR_DEFINE_EXPLICIT_TYPE_ID(cinn::dialect::ConcatOp) diff --git a/paddle/cinn/hlir/dialect/operator/ir/manual_op.h b/paddle/cinn/hlir/dialect/operator/ir/manual_op.h index ba116d52a98c01..acfc7033228f64 100644 --- a/paddle/cinn/hlir/dialect/operator/ir/manual_op.h +++ b/paddle/cinn/hlir/dialect/operator/ir/manual_op.h @@ -44,7 +44,26 @@ class GroupOp : public pir::Op { void Print(pir::IrPrinter &printer); // NOLINT }; +class IR_API ConcatOp : public pir::Op { + public: + using Op::Op; + + static const char *name() { return "cinn_op.concat"; } + + static constexpr uint32_t attributes_num = 1; + + static const char *attributes_name[attributes_num]; + + static void Build(pir::Builder &builder, // NOLINT + pir::OperationArgument &argument, // NOLINT + const std::vector &inputs, + int axis); + + void VerifySig() const {} +}; + } // namespace dialect } // namespace cinn IR_DECLARE_EXPLICIT_TYPE_ID(cinn::dialect::GroupOp) +IR_DECLARE_EXPLICIT_TYPE_ID(cinn::dialect::ConcatOp) diff --git a/paddle/cinn/hlir/dialect/operator/ir/op_dialect.cc b/paddle/cinn/hlir/dialect/operator/ir/op_dialect.cc index 11ccd77bb109d0..4b5f1b82277c9b 100644 --- a/paddle/cinn/hlir/dialect/operator/ir/op_dialect.cc +++ b/paddle/cinn/hlir/dialect/operator/ir/op_dialect.cc @@ -38,6 +38,7 @@ void OperatorDialect::initialize() { #include "paddle/cinn/hlir/dialect/operator/ir/cinn_op.h" // NOLINT >(); RegisterOp(); + RegisterOp(); RegisterAttribute(); RegisterAttribute(); } diff --git a/paddle/cinn/hlir/dialect/operator/ir/ops.yaml b/paddle/cinn/hlir/dialect/operator/ir/ops.yaml index 38fe8674f88fae..015083227e3703 100644 --- a/paddle/cinn/hlir/dialect/operator/ir/ops.yaml +++ b/paddle/cinn/hlir/dialect/operator/ir/ops.yaml @@ -41,6 +41,14 @@ kernel : func : scale +- op : slice + args : (Tensor x, int64_t[] axes, int64_t[] starts, int64_t[] ends, int64_t[] infer_flags, int64_t[] decrease_axis) + output : Tensor + infer_meta : + func : SliceRawInferMeta + kernel : + func : slice + - op : uniform_random args : (int64_t[] shape, float min, float max, int seed, DataType dtype, int diag_num = 0, int diag_step=0, float diag_val=1.0) output : Tensor(out) diff --git a/paddle/cinn/hlir/dialect/operator/transforms/add_broadcast_to_elementwise_pass.cc b/paddle/cinn/hlir/dialect/operator/transforms/add_broadcast_to_elementwise_pass.cc index f7eb951822e075..43594f23eeaced 100644 --- a/paddle/cinn/hlir/dialect/operator/transforms/add_broadcast_to_elementwise_pass.cc +++ b/paddle/cinn/hlir/dialect/operator/transforms/add_broadcast_to_elementwise_pass.cc @@ -148,8 +148,7 @@ bool ProcessOp(pir::Operation* op, pir::PatternRewriter* rewriter) { .dyn_cast() .data()); - rewriter->ReplaceOp(full_op, - std::vector({new_full->result(0)})); + op->operand(1).set_source(new_full->result(0)); } else { auto new_transpose_op = rewriter->Build( op->operand_source(1), diff --git a/paddle/cinn/hlir/dialect/operator/transforms/cinn_group_lowering_pass.cc b/paddle/cinn/hlir/dialect/operator/transforms/cinn_group_lowering_pass.cc index 616f2ef222eef8..7b08187cfe9b6d 100644 --- a/paddle/cinn/hlir/dialect/operator/transforms/cinn_group_lowering_pass.cc +++ b/paddle/cinn/hlir/dialect/operator/transforms/cinn_group_lowering_pass.cc @@ -191,9 +191,8 @@ std::unique_ptr CINNGroupLoweringPass(::pir::Program* program) { for (size_t i = 0; i < cinn_op->num_results(); ++i) { auto find_it = value2id.find(group->output_values[i]); - if (find_it == value2id.end()) { - value_map[group->output_values[i]] = cinn_op->result(i); - } else { + value_map[group->output_values[i]] = cinn_op->result(i); + if (find_it != value2id.end()) { value_map[group_op.result(find_it->second)] = cinn_op->result(i); } } diff --git a/paddle/cinn/hlir/dialect/operator/transforms/op_with_group_merge_pass.cc b/paddle/cinn/hlir/dialect/operator/transforms/op_with_group_merge_pass.cc index 429fb7b6eae938..72f963686275f3 100644 --- a/paddle/cinn/hlir/dialect/operator/transforms/op_with_group_merge_pass.cc +++ b/paddle/cinn/hlir/dialect/operator/transforms/op_with_group_merge_pass.cc @@ -52,7 +52,11 @@ std::unordered_map OpKindMap = { {"pd_op.cast", OpPatternKind::kElementWise}, {"pd_op.greater_than", OpPatternKind::kElementWise}, {"pd_op.greater_equal", OpPatternKind::kElementWise}, + {"pd_op.transpose", OpPatternKind::kInjective}, + {"pd_op.gather_nd", OpPatternKind::kInjective}, {"cinn_op.scale", OpPatternKind::kElementWise}, + {"cinn_op.concat", OpPatternKind::kInjective}, + {"cinn_op.slice", OpPatternKind::kInjective}, {"cinn_op.reduce_sum", OpPatternKind::kReduction}, {"cinn_op.reduce_max", OpPatternKind::kReduction}, {"cinn_op.broadcast", OpPatternKind::kBroadcast}, diff --git a/paddle/cinn/hlir/dialect/operator/transforms/pd_to_cinn_pass.cc b/paddle/cinn/hlir/dialect/operator/transforms/pd_to_cinn_pass.cc index ad8f67341b98ea..6eefa66fbec105 100644 --- a/paddle/cinn/hlir/dialect/operator/transforms/pd_to_cinn_pass.cc +++ b/paddle/cinn/hlir/dialect/operator/transforms/pd_to_cinn_pass.cc @@ -15,6 +15,8 @@ #include "paddle/cinn/hlir/dialect/operator/transforms/pd_to_cinn_pass.h" #include "paddle/cinn/hlir/dialect/operator/ir/cinn_op.h" +#include "paddle/cinn/hlir/dialect/operator/ir/manual_op.h" +#include "paddle/cinn/hlir/dialect/operator/transforms/op_with_group_merge_util.h" #include "paddle/fluid/pir/dialect/operator/ir/pd_op.h" #include "paddle/fluid/pir/drr/api/drr_pattern_base.h" #include "paddle/fluid/pir/drr/api/match_context.h" @@ -104,7 +106,6 @@ class ScaleOpPattern : public pir::OpRewritePattern { .data()); rewriter.ReplaceAllUsesWith(op.result(0), cinn_scale.result(0)); rewriter.EraseOp(op); - rewriter.EraseOp(full_op); } else { // using mul op auto bias = @@ -165,7 +166,79 @@ class ReshapeOpPattern op->operand_source(0).dyn_cast(), vec_out_shape); rewriter.ReplaceAllUsesWith(op.result(0), cinn_reshape.result(0)); rewriter.EraseOp(op); - rewriter.EraseOp(full_op); + + return true; + } + return false; + } +}; + +class SliceOpPattern : public pir::OpRewritePattern { + public: + using pir::OpRewritePattern::OpRewritePattern; + + bool MatchAndRewrite(paddle::dialect::SliceOp op, + pir::PatternRewriter &rewriter) const override { + auto start_gen_op = op->operand_source(1) + .dyn_cast() + .owner() + ->dyn_cast(); + + auto end_gen_op = op->operand_source(2) + .dyn_cast() + .owner() + ->dyn_cast(); + + if (start_gen_op && end_gen_op) { + // sacle is generator by full op + // get attribute value from full op + auto start_vec = cinn::dialect::ir::GetVectorAttr(start_gen_op, "value"); + auto end_vec = cinn::dialect::ir::GetVectorAttr(end_gen_op, "value"); + auto axes = cinn::dialect::ir::GetVectorAttr(op, "axes"); + auto decrease_axis = + cinn::dialect::ir::GetVectorAttr(op, "decrease_axis"); + auto infer_flags = cinn::dialect::ir::GetVectorAttr(op, "infer_flags"); + + auto cinn_slice = rewriter.Build( + op->operand_source(0).dyn_cast(), + axes, + start_vec, + end_vec, + infer_flags, + decrease_axis); + rewriter.ReplaceAllUsesWith(op.result(0), cinn_slice.result(0)); + rewriter.EraseOp(op); + + return true; + } + return false; + } +}; + +class ConcatOpPattern + : public pir::OpRewritePattern { + public: + using pir::OpRewritePattern::OpRewritePattern; + + bool MatchAndRewrite(paddle::dialect::ConcatOp op, + pir::PatternRewriter &rewriter) const override { + auto axis_gen_op = op->operand_source(1).dyn_cast().owner(); + if (auto full_op = axis_gen_op->dyn_cast()) { + int axis = phi::Scalar(full_op.attribute("value") + .dyn_cast<::pir::FloatAttribute>() + .data()) + .to(); + + auto input_ops = op->operand_source(0) + .dyn_cast() + .owner() + ->dyn_cast() + .inputs(); + + auto cinn_concat = + rewriter.Build(input_ops, axis); + rewriter.ReplaceAllUsesWith(op.result(0), cinn_concat.result(0)); + rewriter.EraseOp(op); return true; } @@ -232,6 +305,8 @@ pir::RewritePatternSet PdOpToCinnOpPass::InitializePatterns( ps.Add(SumOpPattern().Build(context)); ps.Add(MaxOpPattern().Build(context)); ps.Add(context); + ps.Add(context); + ps.Add(context); // ps.Add(UniformOpPattern().Build(context)); return ps; diff --git a/paddle/cinn/hlir/framework/pir/op_mapper.cc b/paddle/cinn/hlir/framework/pir/op_mapper.cc index d56dbf240c1e8c..477ceb52613fb1 100644 --- a/paddle/cinn/hlir/framework/pir/op_mapper.cc +++ b/paddle/cinn/hlir/framework/pir/op_mapper.cc @@ -25,55 +25,70 @@ namespace pir { namespace { +std::vector GetVec32FromVec64Attr(::pir::Attribute attr) { + auto attr_vec = attr.dyn_cast<::pir::ArrayAttribute>().AsVector(); + + std::vector dim; + for (auto vec_element : attr_vec) { + dim.push_back(vec_element.dyn_cast<::pir::Int64Attribute>().data()); + } + + return dim; +} + void AppendAttrForReduceOp(const ::pir::Operation& op, utils::AttributeMap& attrs) { // NOLINT auto attr = op.attributes().at("dim"); + attrs["dim"] = GetVec32FromVec64Attr(attr); +} + +void AppendAttrForTransposeOp(const ::pir::Operation& op, + utils::AttributeMap& attrs) { // NOLINT + auto attr = op.attributes().at("perm"); + auto attr_vec = attr.dyn_cast<::pir::ArrayAttribute>().AsVector(); std::vector dim; for (auto vec_element : attr_vec) { - dim.push_back(vec_element.dyn_cast<::pir::Int64Attribute>().data()); + dim.push_back(vec_element.dyn_cast<::pir::Int32Attribute>().data()); } - attrs["dim"] = dim; + attrs["axis"] = dim; } void AppendAttrForUniformOp(const ::pir::Operation& op, utils::AttributeMap& attrs) { // NOLINT auto attr = op.attributes().at("shape"); - auto attr_vec = attr.dyn_cast<::pir::ArrayAttribute>().AsVector(); - - std::vector shape; - for (auto vec_element : attr_vec) { - shape.push_back(vec_element.dyn_cast<::pir::Int64Attribute>().data()); - } - attrs["shape"] = shape; + attrs["shape"] = GetVec32FromVec64Attr(attr); attrs["dtype"] = "float32"; } void AppendAttrForBoadcastToOp(const ::pir::Operation& op, utils::AttributeMap& attrs) { // NOLINT auto axes_attr = op.attributes().at("broadcast_axes"); - auto attr_vec = axes_attr.dyn_cast<::pir::ArrayAttribute>().AsVector(); + attrs["broadcast_axes"] = GetVec32FromVec64Attr(axes_attr); - std::vector axis; - for (auto vec_element : attr_vec) { - axis.push_back(vec_element.dyn_cast<::pir::Int64Attribute>().data()); - } + auto out_shape_attr = op.attributes().at("out_shape"); + attrs["out_shape"] = GetVec32FromVec64Attr(out_shape_attr); +} - attrs["broadcast_axes"] = axis; +void AppendAttrForSliceOp(const ::pir::Operation& op, + utils::AttributeMap& attrs) { // NOLINT + auto axes_attr = op.attributes().at("axes"); + attrs["axes"] = GetVec32FromVec64Attr(axes_attr); - auto out_shape_attr = op.attributes().at("out_shape"); - auto out_shape_attr_vec = - out_shape_attr.dyn_cast<::pir::ArrayAttribute>().AsVector(); + auto starts_attr = op.attributes().at("starts"); + attrs["starts"] = GetVec32FromVec64Attr(starts_attr); - std::vector out_shape; - for (auto vec_element : out_shape_attr_vec) { - out_shape.push_back(vec_element.dyn_cast<::pir::Int64Attribute>().data()); - } + auto ends_attr = op.attributes().at("ends"); + attrs["ends"] = GetVec32FromVec64Attr(ends_attr); + + auto infer_flags_attr = op.attributes().at("infer_flags"); + attrs["infer_flags"] = GetVec32FromVec64Attr(infer_flags_attr); - attrs["out_shape"] = out_shape; + auto decrease_axis_attr = op.attributes().at("decrease_axis"); + attrs["decrease_axis"] = GetVec32FromVec64Attr(decrease_axis_attr); } } // namespace @@ -86,6 +101,9 @@ void AppendAttrForBoadcastToOp(const ::pir::Operation& op, #define REGISTER_ATTR_RULE(OP, func) \ attr_funcs_[cinn::dialect::OP::name()] = func; +#define REGISTER_PD_ATTR_RULE(OP, func) \ + attr_funcs_[paddle::dialect::OP::name()] = func; + void OpMapper::RegisterMapRules() { // max(x, dim) -> reduce_max(x) REGISTER_OPERAND_RULE(MaxOp, 0); @@ -96,6 +114,8 @@ void OpMapper::RegisterMapRules() { REGISTER_ATTR_RULE(ReduceSumOp, AppendAttrForReduceOp); REGISTER_ATTR_RULE(BroadcastOp, AppendAttrForBoadcastToOp); REGISTER_ATTR_RULE(UniformRandomOp, AppendAttrForUniformOp); + REGISTER_PD_ATTR_RULE(TransposeOp, AppendAttrForTransposeOp); + REGISTER_ATTR_RULE(SliceOp, AppendAttrForSliceOp); } } // namespace pir diff --git a/paddle/cinn/hlir/framework/pir/utils.cc b/paddle/cinn/hlir/framework/pir/utils.cc index 0e3c2df6f46e40..a7dfb43991315e 100644 --- a/paddle/cinn/hlir/framework/pir/utils.cc +++ b/paddle/cinn/hlir/framework/pir/utils.cc @@ -168,6 +168,7 @@ utils::Attribute CompatibleInfo::ConvertAttribute( } else if (attr_vec[0].isa<::pir::Int64Attribute>()) { std::vector vec_int64; + int index = 0; for (auto vec_element : attr_vec) { vec_int64.push_back( vec_element.dyn_cast<::pir::Int64Attribute>().data()); diff --git a/paddle/cinn/hlir/framework/pir_compiler.cc b/paddle/cinn/hlir/framework/pir_compiler.cc index 1ad5e921d314a9..0e94f309f5e230 100644 --- a/paddle/cinn/hlir/framework/pir_compiler.cc +++ b/paddle/cinn/hlir/framework/pir_compiler.cc @@ -165,7 +165,7 @@ std::shared_ptr BuildScope(const Target& target, auto scope = std::make_shared(); auto create_var = [&](::pir::Value value) { - if (!(value.type())) { + if (!(value) || !(value.type())) { return; } if (visited.count(value) > 0) return; diff --git a/paddle/fluid/pir/transforms/dead_code_elimination_pass.cc b/paddle/fluid/pir/transforms/dead_code_elimination_pass.cc index f8c18d3d3c9cce..90d378e6c14cc2 100644 --- a/paddle/fluid/pir/transforms/dead_code_elimination_pass.cc +++ b/paddle/fluid/pir/transforms/dead_code_elimination_pass.cc @@ -38,6 +38,10 @@ class DeadCodeEliminationPattern : public pir::RewritePattern { bool Match(pir::Operation* op) const override { if (op->HasTrait()) return false; + + if (op->isa()) { + return false; + } return op->use_empty(); } diff --git a/paddle/fluid/pybind/pir.cc b/paddle/fluid/pybind/pir.cc index e03f6972f5a87a..818db35e55e41f 100644 --- a/paddle/fluid/pybind/pir.cc +++ b/paddle/fluid/pybind/pir.cc @@ -1529,12 +1529,14 @@ std::shared_ptr ApplyPirPass(Program &forward_program) { // NOLINT pass_manager.AddPass( std::make_unique()); + pass_manager.AddPass(pir::CreateDeadCodeEliminationPass()); pass_manager.AddPass(pir::CreateBuildCinnPass()); pass_manager.Run(&forward_program); VLOG(3) << "after BuildCinnPass, forward_program:\n" << forward_program; std::unique_ptr new_program = cinn::dialect::ir::CINNGroupLoweringPass(&forward_program); + VLOG(3) << "after CINNGroupLoweringPass, forward_program:\n" << *new_program; return std::move(new_program); #endif diff --git a/python/paddle/base/variable_index.py b/python/paddle/base/variable_index.py index 41aa40c2574e88..ce55bbad20b70e 100644 --- a/python/paddle/base/variable_index.py +++ b/python/paddle/base/variable_index.py @@ -1212,10 +1212,16 @@ def _getitem_static(x, indices): adjusted_advanced_index = parse_bool_and_broadcast_indices( adjusted_advanced_index ) - advanced_index_tensor = paddle.stack( - adjusted_advanced_index, axis=-1 - ) - out = paddle.gather_nd(transed_tensor, advanced_index_tensor) + + if len(adjusted_advanced_index) > 1: + advanced_index_tensor = paddle.stack( + adjusted_advanced_index, axis=-1 + ) + out = paddle.gather_nd(transed_tensor, advanced_index_tensor) + else: + out = paddle.gather_nd( + transed_tensor, adjusted_advanced_index[0].unsqueeze(-1) + ) if pos_of_new_dim != 0: perm = ( diff --git a/test/cpp/pir/cinn/pir_all_path_test.cc b/test/cpp/pir/cinn/pir_all_path_test.cc index 1e0755b0e54102..098422f2cf237d 100644 --- a/test/cpp/pir/cinn/pir_all_path_test.cc +++ b/test/cpp/pir/cinn/pir_all_path_test.cc @@ -27,6 +27,7 @@ #include "paddle/fluid/pir/dialect/operator/ir/pd_api.h" #include "paddle/fluid/pir/dialect/operator/ir/pd_op.h" #include "paddle/fluid/pir/transforms/build_cinn_pass.h" +#include "paddle/fluid/pir/transforms/dead_code_elimination_pass.h" #include "paddle/fluid/pir/transforms/pd_op_to_kernel_pass.h" #include "paddle/pir/core/builtin_dialect.h" #include "paddle/pir/core/builtin_type.h" @@ -689,3 +690,136 @@ TEST(GroupOp, TestBuildSum2Group) { bool res1 = (out_tensor2.data()[0] == 0.0); EXPECT_EQ(res1, true); } + +std::shared_ptr<::pir::Program> BuildConcatProgram() { + ::pir::IrContext* ctx = ::pir::IrContext::Instance(); + ctx->GetOrRegisterDialect(); + auto program = std::make_shared<::pir::Program>(ctx); + ::pir::Builder builder = ::pir::Builder(ctx, program->block()); + + auto x = builder + .Build(std::vector({16, 16}), + 2.0, + phi::DataType::FLOAT32, + phi::GPUPlace()) + .result(0); + + auto y = builder + .Build(std::vector({16, 16}), + 2.0, + phi::DataType::FLOAT32, + phi::GPUPlace()) + .result(0); + + auto t1 = + builder.Build(std::vector({x, y})).result(0); + + auto out = builder.Build(t1, 1).result(0); + + builder.Build(out, "out", 0); + return program; +} + +TEST(GroupOp, TestBuildConcat) { + // Step 1: Construct pir::Program + ::pir::IrContext* ctx = ::pir::IrContext::Instance(); + std::shared_ptr<::pir::Program> program = BuildConcatProgram(); + ctx->GetOrRegisterDialect(); + ctx->GetOrRegisterDialect(); + + cinn::dialect::ir::PdOp2CinnOpConverter(program.get()); + + pir::PassManager pm(ctx); + pm.AddPass( + std::make_unique()); + + pm.AddPass(pir::CreateDeadCodeEliminationPass()); + pm.AddPass(pir::CreateBuildCinnPass()); + CHECK_EQ(pm.Run(program.get()), true); + + auto res = cinn::dialect::ir::CINNGroupLoweringPass(program.get()); + + paddle::platform::Place place = paddle::platform::CUDAPlace(0); + + auto kernel_program = + paddle::dialect::PdOpLowerToKernelPass(res.get(), place); + + paddle::framework::Scope exe_scope; + + paddle::framework::InterpreterCore executor( + place, {"out@fetch"}, kernel_program->block(), &exe_scope); + + executor.Run({}, true); + + auto out_tensor = + executor.local_scope()->FindVar("out@fetch")->Get(); + + bool res0 = simple_cmp(out_tensor.data()[0], 2.0); + EXPECT_EQ(res0, true); +} + +std::shared_ptr<::pir::Program> BuildSliceProgram() { + ::pir::IrContext* ctx = ::pir::IrContext::Instance(); + ctx->GetOrRegisterDialect(); + auto program = std::make_shared<::pir::Program>(ctx); + ::pir::Builder builder = ::pir::Builder(ctx, program->block()); + + auto x = builder + .Build(std::vector({16, 16}), + 2.0, + phi::DataType::FLOAT32, + phi::GPUPlace()) + .result(0); + + auto out = builder + .Build(x, + std::vector({1}), + std::vector({0}), + std::vector({2}), + std::vector({}), + std::vector({})) + .result(0); + + builder.Build(out, "out", 0); + return program; +} + +// TEST(GroupOp, TestBuildSlice) { +// // Step 1: Construct pir::Program +// ::pir::IrContext* ctx = ::pir::IrContext::Instance(); +// std::shared_ptr<::pir::Program> program = BuildSliceProgram(); +// ctx->GetOrRegisterDialect(); +// ctx->GetOrRegisterDialect(); + +// program->Print(std::cout); +// cinn::dialect::ir::PdOp2CinnOpConverter(program.get()); + +// program->Print(std::cout); +// pir::PassManager pm(ctx); +// pm.AddPass( +// std::make_unique()); + +// pm.AddPass(pir::CreateDeadCodeEliminationPass()); +// pm.AddPass(pir::CreateBuildCinnPass()); +// CHECK_EQ(pm.Run(program.get()), true); + +// auto res = cinn::dialect::ir::CINNGroupLoweringPass(program.get()); + +// paddle::platform::Place place = paddle::platform::CUDAPlace(0); + +// auto kernel_program = +// paddle::dialect::PdOpLowerToKernelPass(res.get(), place); + +// paddle::framework::Scope exe_scope; + +// paddle::framework::InterpreterCore executor( +// place, {"out@fetch"}, kernel_program->block(), &exe_scope); + +// executor.Run({}, true); + +// // auto out_tensor = +// // executor.local_scope()->FindVar("out@fetch")->Get(); + +// // bool res0 = simple_cmp(out_tensor.data()[0], 2.0); +// // EXPECT_EQ(res0, true); +// } diff --git a/test/ir/pir/cinn/test_cinn_sub_graph.py b/test/ir/pir/cinn/test_cinn_sub_graph.py index dcb3bc5c2d5daa..b9e888b8736c75 100644 --- a/test/ir/pir/cinn/test_cinn_sub_graph.py +++ b/test/ir/pir/cinn/test_cinn_sub_graph.py @@ -184,7 +184,6 @@ def train(self, use_cinn): paddle.seed(2022) net = CINNSoftmaxSubGraphNet() net = apply_to_static(net, use_cinn) - out = net(self.x, self.axis) loss = out.mean() diff --git a/test/ir/pir/cinn/test_llama_sub_graph.py b/test/ir/pir/cinn/test_llama_sub_graph.py index 0441b98ec2627c..8393d449898717 100644 --- a/test/ir/pir/cinn/test_llama_sub_graph.py +++ b/test/ir/pir/cinn/test_llama_sub_graph.py @@ -71,6 +71,7 @@ def __init__(self): def forward(self, q, k, cos, sin, position_ids): cos = cos.squeeze(axis=[0, 2]) # [seq_len, dim] sin = sin.squeeze(axis=[0, 2]) # [seq_len, dim] + cos = cos[position_ids].unsqueeze(2) # [bs, seq_len, 1, dim] sin = sin[position_ids].unsqueeze(2) # [bs, seq_len, 1, dim] q_embed = (q * cos) + (self.rotate_half(q) * sin) @@ -104,20 +105,22 @@ def prepare_data(self): def eval(self, use_cinn): paddle.seed(2022) net = RotaryPosEmb() - # TODO(Aurelius84): Need to remove it after verify CINN - if use_cinn: - net = apply_to_static(net, False) net.eval() + if use_cinn: + net = apply_to_static(net, use_cinn) + out = net(self.q, self.k, self.cos, self.sin, self.position_ids) return out def test_eval(self): cinn_outs = self.eval(use_cinn=True) - dy_outs = self.eval(use_cinn=False) - for cinn_out, dy_out in zip(cinn_outs, dy_outs): - np.testing.assert_allclose( - cinn_out.numpy(), dy_out.numpy(), atol=1e-8 - ) + # dy_outs = self.eval(use_cinn=False) + + # TODO(phlrain): Need to check result + # for cinn_out, dy_out in zip(cinn_outs, dy_outs): + # np.testing.assert_allclose( + # cinn_out.numpy(), dy_out.numpy(), atol=1e-8 + # ) class RepeatKV(nn.Layer): From a7fcfbc7df8da2ec962f18c165ce29e42a412fb2 Mon Sep 17 00:00:00 2001 From: Chen Zhiyang <1792266893@qq.com> Date: Tue, 21 Nov 2023 10:04:48 +0800 Subject: [PATCH 34/46] add test time for test_activation_nn_grad (#59151) --- test/legacy_test/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/legacy_test/CMakeLists.txt b/test/legacy_test/CMakeLists.txt index 6e738926020436..7ce8683ed12bf0 100644 --- a/test/legacy_test/CMakeLists.txt +++ b/test/legacy_test/CMakeLists.txt @@ -924,7 +924,7 @@ set_tests_properties(test_inplace_softmax_with_cross_entropy PROPERTIES TIMEOUT set_tests_properties(test_cross_entropy2_op PROPERTIES TIMEOUT 120) set_tests_properties(test_cross_entropy_loss PROPERTIES TIMEOUT 180) set_tests_properties(test_gru_unit_op PROPERTIES TIMEOUT 120) -set_tests_properties(test_activation_nn_grad PROPERTIES TIMEOUT 200) +set_tests_properties(test_activation_nn_grad PROPERTIES TIMEOUT 250) set_tests_properties(test_empty_op PROPERTIES TIMEOUT 120) set_tests_properties(test_parallel_executor_transformer PROPERTIES TIMEOUT 120) set_tests_properties(test_elementwise_div_op PROPERTIES TIMEOUT 120) From 27be0545537fcd24d3c1b20e43b7c611ce17aacd Mon Sep 17 00:00:00 2001 From: hong <43953930+phlrain@users.noreply.github.com> Date: Tue, 21 Nov 2023 10:09:26 +0800 Subject: [PATCH 35/46] polish code (#59162) --- .../operator/transforms/op_with_group_merge_pass.cc | 7 ------- .../dialect/operator/transforms/op_with_group_merge_util.h | 3 +-- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/paddle/cinn/hlir/dialect/operator/transforms/op_with_group_merge_pass.cc b/paddle/cinn/hlir/dialect/operator/transforms/op_with_group_merge_pass.cc index 72f963686275f3..97366302fab009 100644 --- a/paddle/cinn/hlir/dialect/operator/transforms/op_with_group_merge_pass.cc +++ b/paddle/cinn/hlir/dialect/operator/transforms/op_with_group_merge_pass.cc @@ -433,14 +433,7 @@ class OpFusionPassHelper { size_t producer_data_used_num = 0; auto consumer_list = GetConsumerOps(producer, op2id_); - // for (auto it = producer_data.use_begin(); it != - // producer_data.use_end(); - // ++it) { for (auto consumer_op : consumer_list) { - // auto consumer_op = it->owner(); - if (consumer_op->name() == "cf.yield") { - continue; - } producer_data_used_num++; // if fusion group can't find op, can't merge if (consumer_fusion->ops_set.find(consumer_op) == diff --git a/paddle/cinn/hlir/dialect/operator/transforms/op_with_group_merge_util.h b/paddle/cinn/hlir/dialect/operator/transforms/op_with_group_merge_util.h index 5e116f5f413867..62d5f3848bc42e 100644 --- a/paddle/cinn/hlir/dialect/operator/transforms/op_with_group_merge_util.h +++ b/paddle/cinn/hlir/dialect/operator/transforms/op_with_group_merge_util.h @@ -283,7 +283,6 @@ inline bool horizontal_or_vertical_reduce_relation( inline bool horizontal_or_can_inline(::pir::Operation* producer, const std::shared_ptr& consumer) { // horizontal relation. - return true; if (is_horizontal_relation(producer, consumer)) { if (is_same_size(producer, consumer)) { return true; @@ -291,7 +290,7 @@ inline bool horizontal_or_can_inline(::pir::Operation* producer, // if do broadcast, check can compute inline. // return helper->output_ops_set_.count(producer) == 0; // TODO(phlrain): support output op set check - return true; + return false; } } // vertical relation: 1.can compute inline From 831137c89f288febf3042e924bc49f551ca310ec Mon Sep 17 00:00:00 2001 From: wanghuancoder Date: Tue, 21 Nov 2023 10:16:43 +0800 Subject: [PATCH 36/46] convert disttensor for pylayer (#59148) --- paddle/fluid/pybind/eager_py_layer.cc | 60 +++++++++++++++++++++++++++ paddle/fluid/pybind/eager_utils.h | 1 + 2 files changed, 61 insertions(+) diff --git a/paddle/fluid/pybind/eager_py_layer.cc b/paddle/fluid/pybind/eager_py_layer.cc index 94b7de25ed4b49..c35ee606988e3f 100644 --- a/paddle/fluid/pybind/eager_py_layer.cc +++ b/paddle/fluid/pybind/eager_py_layer.cc @@ -172,6 +172,54 @@ PyObject* pylayer_method_apply(PyObject* cls, ctx->forward_input_tensor_is_duplicable.clear(); ctx->forward_input_tensor_is_duplicable.reserve(inputs_size); std::set input_tensorbases; + + const phi::distributed::ProcessMesh* mesh = nullptr; + for (size_t i = 0; i < inputs_size; i++) { + PyObject* obj = nullptr; + if (i >= args_size) { + obj = PyList_GetItem(kwargs_value_list, i - args_size); // NOLINT + } else { + obj = PyTuple_GET_ITEM(args, i); + } + if (PyCheckTensor(obj)) { + paddle::Tensor& tensor = reinterpret_cast(obj)->tensor; + if (tensor.defined() && tensor.is_dist_tensor()) { + mesh = &(std::dynamic_pointer_cast( + tensor.impl()) + ->dist_attr() + .process_mesh()); + } + } else if (PyList_Check(obj)) { + Py_ssize_t len = PyList_Size(obj); + for (Py_ssize_t j = 0; j < len; j++) { + PyObject* o = PyList_GetItem(obj, j); + if (PyCheckTensor(o)) { + paddle::Tensor& tensor = reinterpret_cast(o)->tensor; + if (tensor.defined() && tensor.is_dist_tensor()) { + mesh = &(std::dynamic_pointer_cast( + tensor.impl()) + ->dist_attr() + .process_mesh()); + } + } + } + } else if (PyTuple_Check(obj)) { + Py_ssize_t len = PyTuple_Size(obj); + for (Py_ssize_t j = 0; j < len; j++) { + PyObject* o = PyTuple_GetItem(obj, j); + if (PyCheckTensor(o)) { + paddle::Tensor& tensor = reinterpret_cast(o)->tensor; + if (tensor.defined() && tensor.is_dist_tensor()) { + mesh = &(std::dynamic_pointer_cast( + tensor.impl()) + ->dist_attr() + .process_mesh()); + } + } + } + } + } + for (size_t i = 0; i < inputs_size; i++) { PyObject* obj = nullptr; if (i >= args_size) { @@ -180,6 +228,10 @@ PyObject* pylayer_method_apply(PyObject* cls, obj = PyTuple_GET_ITEM(args, i); } if (PyCheckTensor(obj)) { + if (mesh) { + ConvertToDistTensor(&(reinterpret_cast(obj)->tensor), + mesh); + } input_tensorbases.insert( reinterpret_cast(obj)->tensor.impl().get()); auto autograd_meta = egr::EagerUtils::nullable_autograd_meta( @@ -199,6 +251,10 @@ PyObject* pylayer_method_apply(PyObject* cls, for (Py_ssize_t j = 0; j < len; j++) { PyObject* o = PyList_GetItem(obj, j); if (PyCheckTensor(o)) { + if (mesh) { + ConvertToDistTensor(&(reinterpret_cast(o)->tensor), + mesh); + } input_tensorbases.insert( reinterpret_cast(o)->tensor.impl().get()); tensors.push_back(&(reinterpret_cast(o)->tensor)); @@ -222,6 +278,10 @@ PyObject* pylayer_method_apply(PyObject* cls, for (Py_ssize_t j = 0; j < len; j++) { PyObject* o = PyTuple_GetItem(obj, j); if (PyCheckTensor(o)) { + if (mesh) { + ConvertToDistTensor(&(reinterpret_cast(o)->tensor), + mesh); + } input_tensorbases.insert( reinterpret_cast(o)->tensor.impl().get()); tensors.push_back(&(reinterpret_cast(o)->tensor)); diff --git a/paddle/fluid/pybind/eager_utils.h b/paddle/fluid/pybind/eager_utils.h index 0d53735574180d..fd013913e61083 100644 --- a/paddle/fluid/pybind/eager_utils.h +++ b/paddle/fluid/pybind/eager_utils.h @@ -465,5 +465,6 @@ void ConvertAllInputsToDistTensor(const phi::distributed::ProcessMesh* mesh, DistTensorConverter(mesh).apply(&args...); } +void ConvertToDistTensor(Tensor* x, const phi::distributed::ProcessMesh* mesh); } // namespace pybind } // namespace paddle From 9c15e019847f2c31f479ceceebde923bc4893f7e Mon Sep 17 00:00:00 2001 From: wanghuancoder Date: Tue, 21 Nov 2023 10:50:29 +0800 Subject: [PATCH 37/46] [AutoParallel] reshard record event for time line (#59159) * reshard record event for time line --- .../auto_parallel/reshard/nd_mesh_reshard_function.h | 2 ++ .../auto_parallel/reshard/p_to_r_reshard_function.h | 2 ++ .../auto_parallel/reshard/p_to_s_reshard_function.h | 2 ++ .../auto_parallel/reshard/r_to_p_reshard_function.h | 2 ++ .../auto_parallel/reshard/r_to_s_reshard_function.h | 4 ++++ .../distributed/auto_parallel/reshard/reshard_function.cc | 3 +++ .../core/distributed/auto_parallel/reshard/reshard_function.h | 2 ++ .../auto_parallel/reshard/s_to_r_reshard_function.h | 4 ++++ .../auto_parallel/reshard/s_to_s_reshard_function.h | 2 ++ .../auto_parallel/reshard/same_status_reshard_function.h | 2 ++ 10 files changed, 25 insertions(+) diff --git a/paddle/phi/core/distributed/auto_parallel/reshard/nd_mesh_reshard_function.h b/paddle/phi/core/distributed/auto_parallel/reshard/nd_mesh_reshard_function.h index 169c51899717ee..2e72ffbc319078 100644 --- a/paddle/phi/core/distributed/auto_parallel/reshard/nd_mesh_reshard_function.h +++ b/paddle/phi/core/distributed/auto_parallel/reshard/nd_mesh_reshard_function.h @@ -28,6 +28,8 @@ class SameNdMeshReshardFunction final : public ReshardFunction { const DistTensor& in, const TensorDistAttr& out_dist_attr, DistTensor* out) override; + + std::string Name() override { return "SameNdMeshReshard"; } }; } // namespace distributed diff --git a/paddle/phi/core/distributed/auto_parallel/reshard/p_to_r_reshard_function.h b/paddle/phi/core/distributed/auto_parallel/reshard/p_to_r_reshard_function.h index c1b0c3cd01060f..746baacf25a51e 100644 --- a/paddle/phi/core/distributed/auto_parallel/reshard/p_to_r_reshard_function.h +++ b/paddle/phi/core/distributed/auto_parallel/reshard/p_to_r_reshard_function.h @@ -31,6 +31,8 @@ class PToRReshardFunction final : public ReshardFunction { const DistTensor& in, const TensorDistAttr& out_dist_attr, DistTensor* out) override; + + std::string Name() override { return "PToRReshard"; } }; } // namespace distributed diff --git a/paddle/phi/core/distributed/auto_parallel/reshard/p_to_s_reshard_function.h b/paddle/phi/core/distributed/auto_parallel/reshard/p_to_s_reshard_function.h index 31b094982f16a0..b51b47ce4de7b8 100644 --- a/paddle/phi/core/distributed/auto_parallel/reshard/p_to_s_reshard_function.h +++ b/paddle/phi/core/distributed/auto_parallel/reshard/p_to_s_reshard_function.h @@ -27,6 +27,8 @@ class PToSReshardFunction final : public ReshardFunction { const DistTensor& in, const TensorDistAttr& out_dist_attr, DistTensor* out) override; + + std::string Name() override { return "PToSReshard"; } }; } // namespace distributed diff --git a/paddle/phi/core/distributed/auto_parallel/reshard/r_to_p_reshard_function.h b/paddle/phi/core/distributed/auto_parallel/reshard/r_to_p_reshard_function.h index 3014cdc550e6c5..04017a1e80baa2 100644 --- a/paddle/phi/core/distributed/auto_parallel/reshard/r_to_p_reshard_function.h +++ b/paddle/phi/core/distributed/auto_parallel/reshard/r_to_p_reshard_function.h @@ -28,6 +28,8 @@ class RToPReshardFunction final : public ReshardFunction { const DistTensor& in, const TensorDistAttr& out_dist_attr, DistTensor* out) override; + + std::string Name() override { return "RToPReshard"; } }; } // namespace distributed diff --git a/paddle/phi/core/distributed/auto_parallel/reshard/r_to_s_reshard_function.h b/paddle/phi/core/distributed/auto_parallel/reshard/r_to_s_reshard_function.h index 4ca086525b0d2a..04ab4e7f954638 100644 --- a/paddle/phi/core/distributed/auto_parallel/reshard/r_to_s_reshard_function.h +++ b/paddle/phi/core/distributed/auto_parallel/reshard/r_to_s_reshard_function.h @@ -28,6 +28,8 @@ class RToSReshardFunction final : public ReshardFunction { const DistTensor& in, const TensorDistAttr& out_dist_attr, DistTensor* out) override; + + std::string Name() override { return "RToSReshard"; } }; class RToSReshardFunctionCrossMesh final : public ReshardFunction { @@ -39,6 +41,8 @@ class RToSReshardFunctionCrossMesh final : public ReshardFunction { const DistTensor& in, const TensorDistAttr& out_dist_attr, DistTensor* out) override; + + std::string Name() override { return "RToSReshardCrossMesh"; } }; } // namespace distributed diff --git a/paddle/phi/core/distributed/auto_parallel/reshard/reshard_function.cc b/paddle/phi/core/distributed/auto_parallel/reshard/reshard_function.cc index 04d47e4151d8a4..8a7d0e95400b59 100644 --- a/paddle/phi/core/distributed/auto_parallel/reshard/reshard_function.cc +++ b/paddle/phi/core/distributed/auto_parallel/reshard/reshard_function.cc @@ -14,6 +14,7 @@ #include "paddle/phi/core/distributed/auto_parallel/reshard/reshard_function.h" +#include "paddle/phi/api/profiler/event_tracing.h" #include "paddle/phi/core/distributed/auto_parallel/dist_attr.h" #include "paddle/phi/core/distributed/auto_parallel/dist_tensor.h" @@ -24,6 +25,8 @@ std::shared_ptr ReshardFunction::Eval( DeviceContext* dev_ctx, const DistTensor& in, const TensorDistAttr& out_dist_attr) { + phi::RecordEvent reshard_record_event( + Name(), phi::TracerEventType::OperatorInner, 1); std::shared_ptr out = std::make_shared(); Eval(dev_ctx, in, out_dist_attr, out.get()); return out; diff --git a/paddle/phi/core/distributed/auto_parallel/reshard/reshard_function.h b/paddle/phi/core/distributed/auto_parallel/reshard/reshard_function.h index dd51768053bbf6..19909ef0a328f3 100644 --- a/paddle/phi/core/distributed/auto_parallel/reshard/reshard_function.h +++ b/paddle/phi/core/distributed/auto_parallel/reshard/reshard_function.h @@ -43,6 +43,8 @@ class ReshardFunction { const TensorDistAttr& out_dist_attr, DistTensor* out) = 0; + virtual std::string Name() { return "ReshardBase"; } + protected: void SetValue(DistTensor* tensor, const DenseTensor& value); void SetDistProps(DistTensor* tensor, diff --git a/paddle/phi/core/distributed/auto_parallel/reshard/s_to_r_reshard_function.h b/paddle/phi/core/distributed/auto_parallel/reshard/s_to_r_reshard_function.h index ee4b65fade96e1..784950a7dfb7f9 100644 --- a/paddle/phi/core/distributed/auto_parallel/reshard/s_to_r_reshard_function.h +++ b/paddle/phi/core/distributed/auto_parallel/reshard/s_to_r_reshard_function.h @@ -31,6 +31,8 @@ class SToRReshardFunction final : public ReshardFunction { const DistTensor& in, const TensorDistAttr& out_dist_attr, DistTensor* out) override; + + std::string Name() override { return "SToRReshard"; } }; class SToRReshardFunctionCrossMesh final : public ReshardFunction { @@ -42,6 +44,8 @@ class SToRReshardFunctionCrossMesh final : public ReshardFunction { const DistTensor& in, const TensorDistAttr& out_dist_attr, DistTensor* out) override; + + std::string Name() override { return "SToRReshardCrossMesh"; } }; } // namespace distributed diff --git a/paddle/phi/core/distributed/auto_parallel/reshard/s_to_s_reshard_function.h b/paddle/phi/core/distributed/auto_parallel/reshard/s_to_s_reshard_function.h index 383c7b522ad62b..43777a99f32fa2 100644 --- a/paddle/phi/core/distributed/auto_parallel/reshard/s_to_s_reshard_function.h +++ b/paddle/phi/core/distributed/auto_parallel/reshard/s_to_s_reshard_function.h @@ -31,6 +31,8 @@ class SToSReshardFunction final : public ReshardFunction { const DistTensor& in, const TensorDistAttr& out_dist_attr, DistTensor* out) override; + + std::string Name() override { return "SToSReshard"; } }; } // namespace distributed diff --git a/paddle/phi/core/distributed/auto_parallel/reshard/same_status_reshard_function.h b/paddle/phi/core/distributed/auto_parallel/reshard/same_status_reshard_function.h index 7abaec5e8f6c39..1b6576e7e6859e 100644 --- a/paddle/phi/core/distributed/auto_parallel/reshard/same_status_reshard_function.h +++ b/paddle/phi/core/distributed/auto_parallel/reshard/same_status_reshard_function.h @@ -28,6 +28,8 @@ class SameStatusReshardFunction final : public ReshardFunction { const DistTensor& in, const TensorDistAttr& out_dist_attr, DistTensor* out) override; + + std::string Name() override { return "SameStatusReshard"; } }; } // namespace distributed From ccb30ddcf071cfa2002cdcbdf2c5c6c40aa3a1d6 Mon Sep 17 00:00:00 2001 From: winter-wang <78149749+winter-wang@users.noreply.github.com> Date: Tue, 21 Nov 2023 11:04:37 +0800 Subject: [PATCH 38/46] [PIR] polish the backward tool op in control flow dialect. (#59175) --- .../pir/transforms/pd_op_to_kernel_pass.cc | 14 +- paddle/pir/core/op_base.h | 2 +- .../pir/dialect/control_flow/ir/cf_dialect.cc | 4 +- .../pir/dialect/control_flow/ir/cf_dialect.h | 6 +- .../dialect/control_flow/ir/cf_interface.cc | 34 +++ .../dialect/control_flow/ir/cf_interface.h | 88 ++++++++ paddle/pir/dialect/control_flow/ir/cf_op.cc | 207 +++++++++--------- paddle/pir/dialect/control_flow/ir/cf_op.h | 96 ++++---- paddle/pir/dialect/control_flow/ir/cf_type.cc | 5 + paddle/pir/dialect/control_flow/ir/cf_type.h | 10 +- .../pir/control_flow_dialect/if_op_test.cc | 18 +- .../pir/control_flow_dialect/while_op_test.cc | 6 +- 12 files changed, 318 insertions(+), 172 deletions(-) create mode 100644 paddle/pir/dialect/control_flow/ir/cf_interface.cc create mode 100644 paddle/pir/dialect/control_flow/ir/cf_interface.h diff --git a/paddle/fluid/pir/transforms/pd_op_to_kernel_pass.cc b/paddle/fluid/pir/transforms/pd_op_to_kernel_pass.cc index 6f1ab3b705cb11..7fbcdc29bfe5c4 100644 --- a/paddle/fluid/pir/transforms/pd_op_to_kernel_pass.cc +++ b/paddle/fluid/pir/transforms/pd_op_to_kernel_pass.cc @@ -78,9 +78,9 @@ const std::unordered_set SpecialLowerOps = { pir::YieldOp::name(), IfOp::name(), WhileOp::name(), - pir::CreateStackOp::name(), - pir::PushBackOp::name(), - pir::PopBackOp::name(), + pir::StackCreateOp::name(), + pir::TuplePushOp::name(), + pir::TuplePopOp::name(), "cinn_runtime.jit_kernel"}; static bool NeedFallBackCpu(const pir::Operation* op, @@ -1038,8 +1038,8 @@ void HandleForSpecialOp( } } - if (op_item->isa<::pir::CreateStackOp>() || - op_item->isa<::pir::PushBackOp>()) { + if (op_item->isa<::pir::StackCreateOp>() || + op_item->isa<::pir::TuplePushOp>()) { for (size_t i = 0; i < op_item->num_operands(); ++i) { auto cur_in = op_item->operand_source(i); if (!cur_in) { @@ -1055,7 +1055,7 @@ void HandleForSpecialOp( } } - if (op_item->isa<::pir::PopBackOp>()) { + if (op_item->isa<::pir::TuplePopOp>()) { for (size_t i = 0; i < op_item->num_operands(); ++i) { auto cur_in = op_item->operand_source(i); auto new_in = GetNewInput( @@ -1063,7 +1063,7 @@ void HandleForSpecialOp( vec_inputs.push_back(new_in); } - auto pop_back_op = op_item->dyn_cast<::pir::PopBackOp>(); + auto pop_back_op = op_item->dyn_cast<::pir::TuplePopOp>(); for (size_t i = 0; i < op_item->num_results(); ++i) { auto cur_inlet_element = pop_back_op.inlet_element(i); PADDLE_ENFORCE_EQ(map_value_pair->count(cur_inlet_element), diff --git a/paddle/pir/core/op_base.h b/paddle/pir/core/op_base.h index d827a7afb476c3..3d8a6509051bd2 100644 --- a/paddle/pir/core/op_base.h +++ b/paddle/pir/core/op_base.h @@ -70,7 +70,7 @@ class IR_API OpBase { void VerifyRegion() {} - private: + protected: Operation *operation_; // Not owned }; diff --git a/paddle/pir/dialect/control_flow/ir/cf_dialect.cc b/paddle/pir/dialect/control_flow/ir/cf_dialect.cc index b10df41168a275..78e18f4cce108d 100644 --- a/paddle/pir/dialect/control_flow/ir/cf_dialect.cc +++ b/paddle/pir/dialect/control_flow/ir/cf_dialect.cc @@ -19,7 +19,7 @@ namespace pir { void ControlFlowDialect::initialize() { RegisterTypes(); - RegisterOps(); + RegisterOps(); } void ControlFlowDialect::PrintType(pir::Type type, std::ostream &os) const { @@ -38,7 +38,7 @@ void ControlFlowDialect::PrintType(pir::Type type, std::ostream &os) const { void ControlFlowDialect::PrintOperation(pir::Operation *op, pir::IrPrinter &printer) const { - if (auto create_op = op->dyn_cast()) { + if (auto create_op = op->dyn_cast()) { create_op.Print(printer); } else { printer.PrintGeneralOperation(op); diff --git a/paddle/pir/dialect/control_flow/ir/cf_dialect.h b/paddle/pir/dialect/control_flow/ir/cf_dialect.h index a319bd888a65f9..f4a347116cda5c 100644 --- a/paddle/pir/dialect/control_flow/ir/cf_dialect.h +++ b/paddle/pir/dialect/control_flow/ir/cf_dialect.h @@ -24,9 +24,9 @@ class ControlFlowDialect : public Dialect { initialize(); } static const char *name() { return "cf"; } - void PrintType(pir::Type type, std::ostream &os) const override; - void PrintOperation(pir::Operation *op, - pir::IrPrinter &printer) const override; // NOLINT + void PrintType(Type type, std::ostream &os) const override; + void PrintOperation(Operation *op, + IrPrinter &printer) const override; // NOLINT private: void initialize(); }; diff --git a/paddle/pir/dialect/control_flow/ir/cf_interface.cc b/paddle/pir/dialect/control_flow/ir/cf_interface.cc new file mode 100644 index 00000000000000..fca199367dc5f9 --- /dev/null +++ b/paddle/pir/dialect/control_flow/ir/cf_interface.cc @@ -0,0 +1,34 @@ +// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/pir/dialect/control_flow/ir/cf_interface.h" +#include "paddle/pir/dialect/control_flow/ir/cf_op.h" + +namespace pir { +TuplePushOp ContainerOpInterface::tuple_push_op() { + auto value = inlet(); + IR_ENFORCE(value.HasOneUse(), + "The inlet value of container op can only be used once."); + return value.first_use().owner()->dyn_cast(); +} +TuplePopOp ContainerOpInterface::tuple_pop_op() { + auto value = outlet(); + IR_ENFORCE(value.HasOneUse(), + "The outlet value of container op can only be used once."); + return value.first_use().owner()->dyn_cast(); +} + +} // namespace pir + +IR_DEFINE_EXPLICIT_TYPE_ID(pir::ContainerOpInterface) diff --git a/paddle/pir/dialect/control_flow/ir/cf_interface.h b/paddle/pir/dialect/control_flow/ir/cf_interface.h new file mode 100644 index 00000000000000..7237eda6064617 --- /dev/null +++ b/paddle/pir/dialect/control_flow/ir/cf_interface.h @@ -0,0 +1,88 @@ +// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include "paddle/pir/core/op_base.h" + +namespace pir { + +class TuplePushOp; +class TuplePopOp; +/// +/// \brief This interface marks the op can create a container. +/// +class ContainerOpInterface : public OpInterfaceBase { + public: + struct Concept { + Value (*container_)(Operation* op); + Value (*inlet_)(Operation* op); + Value (*outlet_)(Operation* op); + size_t (*tuple_size_)(Operation* op); + Value (*inlet_element_)(Operation* op, size_t index); + Value (*outlet_element_)(Operation* op, size_t index); + }; + + template + struct Model : public Concept { + Model() + : Concept{container, + inlet, + outlet, + tuple_size, + inlet_element, + inlet_element} {} + static Value container(Operation* op) { + return op->dyn_cast().container(); + } + static Value inlet(Operation* op) { + return op->dyn_cast().inlet(); + } + static Value outlet(Operation* op) { + return op->dyn_cast().outlet(); + } + static size_t tuple_size(Operation* op) { + return op->dyn_cast().tuple_size(); + } + static Value inlet_element(Operation* op, size_t index) { + return op->dyn_cast().container(); + } + static Value outlet_element(Operation* op, size_t index) { + return op->dyn_cast().container(); + } + }; + + Value container() { return impl_->container_(operation()); } + Value inlet() { return impl_->inlet_(operation()); } + Value outlet() { return impl_->outlet_(operation()); } + size_t tuple_size() { return impl_->tuple_size_(operation()); } + Value inlet_element(size_t index) { + return impl_->inlet_element_(operation(), index); + } + Value outlet_element(size_t index) { + return impl_->outlet_element_(operation(), index); + } + + TuplePushOp tuple_push_op(); + TuplePopOp tuple_pop_op(); + /// Constructor + ContainerOpInterface(pir::Operation* op, Concept* impl) + : OpInterfaceBase(op), impl_(impl) {} + + private: + Concept* impl_; +}; +} // namespace pir + +IR_EXPORT_DECLARE_EXPLICIT_TYPE_ID(pir::ContainerOpInterface) diff --git a/paddle/pir/dialect/control_flow/ir/cf_op.cc b/paddle/pir/dialect/control_flow/ir/cf_op.cc index 6828701d5961d9..f47426414f22d5 100644 --- a/paddle/pir/dialect/control_flow/ir/cf_op.cc +++ b/paddle/pir/dialect/control_flow/ir/cf_op.cc @@ -25,147 +25,84 @@ void YieldOp::Build(Builder &builder, argument.AddInputs(inputs); } -void CreateStackOp::Build(Builder &builder, OperationArgument &argument) { - auto stack_type = StackType::get(builder.ir_context()); - auto inlet_type = InletType::get(builder.ir_context()); - auto outlet_type = OutletType::get(builder.ir_context()); - argument.AddOutputs({stack_type, inlet_type, outlet_type}); -} - -void CreateStackOp::VerifySig() { - VLOG(4) << "Verifying inputs, outputs and attributes for: CreateStackOp."; - // Verify inputs: - IR_ENFORCE(num_operands() == 0u, "The size of inputs must be equal to 0."); - - // No attributes should be verify. - - // Verify outputs: - IR_ENFORCE(num_results() == 3u, "The size of outputs must be equal to 3."); - - IR_ENFORCE(result(0).type().isa(), - "The first outputs of cf.create_stack must be stack_type."); - IR_ENFORCE(result(1).type().isa(), - "The first outputs of cf.create_stack must be inlet_type."); - IR_ENFORCE(result(2).type().isa(), - "The first outputs of cf.create_stack must be outlet_type."); - - VLOG(4) << "End Verifying for CreateStackOp."; -} - -size_t CreateStackOp::stack_size() { return push_op().stack_size(); } - -Value CreateStackOp::inlet_element(size_t index) { - return push_op().inlet_element(index); -} - -Value CreateStackOp::outlet_element(size_t index) { - return pop_op().outlet_element(index); -} - -PushBackOp CreateStackOp::push_op() { - auto inlet_value = inlet(); - IR_ENFORCE(inlet_value.HasOneUse(), "The inlet value must has one use."); - return inlet_value.first_use().owner()->dyn_cast(); -} - -PopBackOp CreateStackOp::pop_op() { - auto outlet_value = outlet(); - IR_ENFORCE(outlet_value.HasOneUse(), "The outlet value must has one use."); - return outlet_value.first_use().owner()->dyn_cast(); -} - -void CreateStackOp::Print(IrPrinter &printer) { // NOLINT - static std::unordered_map> - kConunters; - auto &counter = kConunters[&printer]; - auto iter = counter.insert({*this, counter.size()}); - auto index = iter.first->second; - if (iter.second) { - printer.AddValueAlias(stack(), "%stack_" + std::to_string(index)); - printer.AddValueAlias(inlet(), "%inlet_" + std::to_string(index)); - printer.AddValueAlias(outlet(), "%outlet_" + std::to_string(index)); - } - printer.PrintGeneralOperation(*this); -} - -void PushBackOp::Build(Builder &builder, // NOLINT - OperationArgument &argument, // NOLINT - Value inlet, - const std::vector &elements) { +void TuplePushOp::Build(Builder &builder, // NOLINT + OperationArgument &argument, // NOLINT + Value inlet, + const std::vector &elements) { argument.AddInput(inlet); argument.AddInputs(elements); } -void PushBackOp::Build(Builder &builder, // NOLINT - OperationArgument &argument, // NOLINT - Value inlet, - std::initializer_list element_list) { +void TuplePushOp::Build(Builder &builder, // NOLINT + OperationArgument &argument, // NOLINT + Value inlet, + std::initializer_list element_list) { argument.AddInput(inlet); argument.AddInputs(element_list); } -void PushBackOp::VerifySig() { - VLOG(4) << "Verifying inputs, outputs ,attributes for: PushBackOp."; +void TuplePushOp::VerifySig() { + VLOG(4) << "Verifying inputs, outputs ,attributes for: TuplePushOp."; // Verify inputs: - IR_ENFORCE(num_operands() >= 2u, "The size of inputs must no less than 2."); + IR_ENFORCE(num_operands() >= 1u, "The size of inputs must no less than 1."); IR_ENFORCE(operand_source(0).type().isa(), - "The first input of cf.push_back must be inlet_type."); + "The first input of cf.tuple_push must be inlet_type."); IR_ENFORCE(operand_source(0).HasOneUse(), - "The inlet value of cf.push_back can only be used once."); + "The inlet value of cf.tuple_push can only be used once."); // No attributes should be verify. // Verify outputs: IR_ENFORCE(num_results() == 0u, "The size of outputs must be equal to 0."); - VLOG(4) << "End Verifying for PushBackOp."; + VLOG(4) << "End Verifying for TuplePushOp."; } -size_t PushBackOp::stack_size() { +size_t TuplePushOp::tuple_size() { auto operands_size = num_operands(); - IR_ENFORCE(operands_size >= 2u, - "The operands of push op must no less than 2."); + IR_ENFORCE(operands_size >= 1u, + "The operands of push op must no less than 1."); return operands_size - 1u; } -PopBackOp PushBackOp::pop_op() { return create_op().pop_op(); } +TuplePopOp TuplePushOp::tuple_pop_op() { + return container_interface().tuple_pop_op(); +} -void PopBackOp::Build(Builder &builder, // NOLINT - OperationArgument &argument, // NOLINT - Value outlet) { +void TuplePopOp::Build(Builder &builder, // NOLINT + OperationArgument &argument, // NOLINT + Value outlet) { argument.AddInput(outlet); - auto push_back_op = outlet.defining_op().push_op(); + auto push_op = outlet.defining_op().tuple_push_op(); - auto elements_size = push_back_op.stack_size(); + auto elements_size = push_op.tuple_size(); for (size_t index = 0; index < elements_size; ++index) { - argument.AddOutput(push_back_op.inlet_element(index).type()); + argument.AddOutput(push_op.inlet_element(index).type()); } } -void PopBackOp::VerifySig() { +void TuplePopOp::VerifySig() { VLOG(4) << "Verifying inputs, outputs ,attributes and stack validity for: " - "PopBackOp."; + "TuplePopOp."; // Verify inputs: IR_ENFORCE(num_operands() == 1u, "The size of inputs must equal to 1."); IR_ENFORCE(operand_source(0).type().isa(), - "The first input of cf.pop_back must be outlet_type."); + "The first input of cf.tuple_pop must be outlet_type."); IR_ENFORCE(operand_source(0).HasOneUse(), - "The outlet value of cf.pop_back can only be used once."); + "The outlet value of cf.tuple_pop can only be used once."); // No attributes should be verify. // Verify outputs: - IR_ENFORCE(num_results() >= 1u, - "The size of outputs must no less than to 1."); + // Verify stack validity: - auto pop_back_op = create_op().pop_op(); - IR_ENFORCE(*this == pop_back_op, - "The pop_op of stack_op must be this pop_op self."); + auto pop_op = container_interface().tuple_pop_op(); + IR_ENFORCE(*this == pop_op, + "The pop_op of tuple_pop_op must be this tuple_pop_op self."); - auto inlet_size = push_op().stack_size(); - IR_ENFORCE(inlet_size == stack_size(), + auto inlet_size = tuple_push_op().tuple_size(); + IR_ENFORCE(inlet_size == tuple_size(), "The pop elements size must equal to push elements size."); for (size_t index = 0; index < inlet_size; ++index) { IR_ENFORCE(outlet_element(index).type() == inlet_element(index).type(), @@ -174,7 +111,7 @@ void PopBackOp::VerifySig() { outlet_element(index).type(), inlet_element(index).type()); } - VLOG(4) << "End Verifying for PopBackOp."; + VLOG(4) << "End Verifying for TuplePopOp."; } void HasElementsOp::Build(Builder &builder, // NOLINT @@ -198,10 +135,74 @@ void HasElementsOp::VerifySig() { "The type of cf.has_elements' output is not correct."); } +void StackCreateOp::Build(Builder &builder, OperationArgument &argument) { + auto stack_type = StackType::get(builder.ir_context()); + auto inlet_type = InletType::get(builder.ir_context()); + auto outlet_type = OutletType::get(builder.ir_context()); + argument.AddOutputs({stack_type, inlet_type, outlet_type}); +} + +void StackCreateOp::VerifySig() { + VLOG(4) << "Verifying inputs, outputs and attributes for: StackCreateOp."; + // Verify inputs: + IR_ENFORCE(num_operands() == 0u, "The size of inputs must be equal to 0."); + + // No attributes should be verify. + + // Verify outputs: + IR_ENFORCE(num_results() == 3u, "The size of outputs must be equal to 3."); + + IR_ENFORCE(result(0).type().isa(), + "The first outputs of cf.stack_create must be stack_type."); + IR_ENFORCE(result(1).type().isa(), + "The first outputs of cf.stack_create must be inlet_type."); + IR_ENFORCE(result(2).type().isa(), + "The first outputs of cf.stack_create must be outlet_type."); + + VLOG(4) << "End Verifying for StackCreateOp."; +} + +size_t StackCreateOp::tuple_size() { return tuple_push_op().tuple_size(); } + +Value StackCreateOp::inlet_element(size_t index) { + return tuple_push_op().inlet_element(index); +} + +Value StackCreateOp::outlet_element(size_t index) { + return tuple_pop_op().outlet_element(index); +} + +TuplePushOp StackCreateOp::tuple_push_op() { + auto inlet_value = inlet(); + IR_ENFORCE(inlet_value.HasOneUse(), "The inlet value must has one use."); + return inlet_value.first_use().owner()->dyn_cast(); +} + +TuplePopOp StackCreateOp::tuple_pop_op() { + auto outlet_value = outlet(); + IR_ENFORCE(outlet_value.HasOneUse(), "The outlet value must has one use."); + return outlet_value.first_use().owner()->dyn_cast(); +} + +void StackCreateOp::Print(IrPrinter &printer) { // NOLINT + static std::unordered_map> + kConunters; + auto &counter = kConunters[&printer]; + auto iter = counter.insert({*this, counter.size()}); + auto index = iter.first->second; + if (iter.second) { + printer.AddValueAlias(stack(), "%stack_" + std::to_string(index)); + printer.AddValueAlias(inlet(), "%inlet_" + std::to_string(index)); + printer.AddValueAlias(outlet(), "%outlet_" + std::to_string(index)); + } + printer.PrintGeneralOperation(*this); +} + } // namespace pir IR_DEFINE_EXPLICIT_TYPE_ID(pir::YieldOp) -IR_DEFINE_EXPLICIT_TYPE_ID(pir::CreateStackOp) -IR_DEFINE_EXPLICIT_TYPE_ID(pir::PushBackOp) -IR_DEFINE_EXPLICIT_TYPE_ID(pir::PopBackOp) +IR_DEFINE_EXPLICIT_TYPE_ID(pir::StackCreateOp) +IR_DEFINE_EXPLICIT_TYPE_ID(pir::TuplePushOp) +IR_DEFINE_EXPLICIT_TYPE_ID(pir::TuplePopOp) IR_DEFINE_EXPLICIT_TYPE_ID(pir::HasElementsOp) diff --git a/paddle/pir/dialect/control_flow/ir/cf_op.h b/paddle/pir/dialect/control_flow/ir/cf_op.h index 1f79d747cb42cd..a4fd0af651dc90 100644 --- a/paddle/pir/dialect/control_flow/ir/cf_op.h +++ b/paddle/pir/dialect/control_flow/ir/cf_op.h @@ -17,6 +17,7 @@ #include "paddle/pir/core/builder.h" #include "paddle/pir/core/op_base.h" #include "paddle/pir/core/op_trait.h" +#include "paddle/pir/dialect/control_flow/ir/cf_interface.h" namespace pir { class IR_API YieldOp : public Op { @@ -31,36 +32,14 @@ class IR_API YieldOp : public Op { const std::vector &Value); void VerifySig() {} }; -class PushBackOp; -class PopBackOp; -class IR_API CreateStackOp : public Op { - public: - using Op::Op; - static const char *name() { return "cf.create_stack"; } - static constexpr uint32_t attributes_num = 0; - static constexpr const char **attributes_name = nullptr; - static void Build(Builder &builder, // NOLINT - OperationArgument &argument); // NOLINT - void VerifySig(); - Value stack() { return result(0); } - Value inlet() { return result(1); } - Value outlet() { return result(2); } - std::tuple out() { return {stack(), inlet(), outlet()}; } - - size_t stack_size(); - Value inlet_element(size_t index); - Value outlet_element(size_t index); - PushBackOp push_op(); - PopBackOp pop_op(); - - void Print(pir::IrPrinter &printer); // NOLINT -}; - -class IR_API PushBackOp : public Op { +/// +/// \brief Push a value tuple to a container. +/// +class IR_API TuplePushOp : public Op { public: using Op::Op; - static const char *name() { return "cf.push_back"; } + static const char *name() { return "cf.tuple_push"; } static constexpr uint32_t attributes_num = 0; static constexpr const char **attributes_name = nullptr; @@ -74,22 +53,24 @@ class IR_API PushBackOp : public Op { std::initializer_list element_list); void VerifySig(); - Value stack() { return create_op().stack(); } + Value container() { return container_interface().container(); } Value inlet() { return operand_source(0); } - Value outlet() { return create_op().outlet(); } - size_t stack_size(); + Value outlet() { return container_interface().outlet(); } + size_t tuple_size(); Value inlet_element(size_t index) { return operand_source(index + 1u); } Value outlet_element(size_t index) { - return create_op().outlet_element(index); + return container_interface().outlet_element(index); + } + ContainerOpInterface container_interface() { + return inlet().defining_op(); } - CreateStackOp create_op() { return inlet().defining_op(); } - PopBackOp pop_op(); + TuplePopOp tuple_pop_op(); }; -class IR_API PopBackOp : public Op { +class IR_API TuplePopOp : public Op { public: using Op::Op; - static const char *name() { return "cf.pop_back"; } + static const char *name() { return "cf.tuple_pop"; } static constexpr uint32_t attributes_num = 0; static constexpr const char **attributes_name = nullptr; @@ -98,15 +79,19 @@ class IR_API PopBackOp : public Op { Value outlet); void VerifySig(); - Value stack() { return create_op().stack(); } - Value inlet() { return create_op().inlet(); } + Value container() { return container_interface().container(); } + Value inlet() { return container_interface().inlet(); } Value outlet() { return operand_source(0); } - size_t stack_size() { return num_results(); } - Value inlet_element(size_t index) { return push_op().inlet_element(index); } + size_t tuple_size() { return num_results(); } + Value inlet_element(size_t index) { + return tuple_push_op().inlet_element(index); + } Value outlet_element(size_t index) { return result(index); } - CreateStackOp create_op() { return outlet().defining_op(); } - PushBackOp push_op() { return create_op().push_op(); } + ContainerOpInterface container_interface() { + return outlet().defining_op(); + } + TuplePushOp tuple_push_op() { return container_interface().tuple_push_op(); } }; class IR_API HasElementsOp : public Op { @@ -122,11 +107,34 @@ class IR_API HasElementsOp : public Op { void VerifySig(); Value out() { return result(0); } }; +class IR_API StackCreateOp : public Op { + public: + using Op::Op; + static const char *name() { return "cf.stack_create"; } + static constexpr uint32_t attributes_num = 0; + static constexpr const char **attributes_name = nullptr; + static void Build(Builder &builder, // NOLINT + OperationArgument &argument); // NOLINT + void VerifySig(); + Value container() { return result(0); } + Value stack() { return result(0); } + Value inlet() { return result(1); } + Value outlet() { return result(2); } + std::tuple out() { return {stack(), inlet(), outlet()}; } + + size_t tuple_size(); + Value inlet_element(size_t index); + Value outlet_element(size_t index); + TuplePushOp tuple_push_op(); + TuplePopOp tuple_pop_op(); + + void Print(pir::IrPrinter &printer); // NOLINT +}; } // namespace pir IR_EXPORT_DECLARE_EXPLICIT_TYPE_ID(pir::YieldOp); -IR_EXPORT_DECLARE_EXPLICIT_TYPE_ID(pir::CreateStackOp); -IR_EXPORT_DECLARE_EXPLICIT_TYPE_ID(pir::PushBackOp); -IR_EXPORT_DECLARE_EXPLICIT_TYPE_ID(pir::PopBackOp); +IR_EXPORT_DECLARE_EXPLICIT_TYPE_ID(pir::StackCreateOp); +IR_EXPORT_DECLARE_EXPLICIT_TYPE_ID(pir::TuplePushOp); +IR_EXPORT_DECLARE_EXPLICIT_TYPE_ID(pir::TuplePopOp); IR_EXPORT_DECLARE_EXPLICIT_TYPE_ID(pir::HasElementsOp); diff --git a/paddle/pir/dialect/control_flow/ir/cf_type.cc b/paddle/pir/dialect/control_flow/ir/cf_type.cc index 19ec9af3864e35..baa3fd61d2a1da 100644 --- a/paddle/pir/dialect/control_flow/ir/cf_type.cc +++ b/paddle/pir/dialect/control_flow/ir/cf_type.cc @@ -14,6 +14,11 @@ #include "paddle/pir/dialect/control_flow/ir/cf_type.h" +namespace pir { +bool ContainerType::classof(Type type) { return StackType::classof(type); } + +} // namespace pir +IR_DEFINE_EXPLICIT_TYPE_ID(pir::ContainerType) IR_DEFINE_EXPLICIT_TYPE_ID(pir::StackType) IR_DEFINE_EXPLICIT_TYPE_ID(pir::InletType) IR_DEFINE_EXPLICIT_TYPE_ID(pir::OutletType) diff --git a/paddle/pir/dialect/control_flow/ir/cf_type.h b/paddle/pir/dialect/control_flow/ir/cf_type.h index 6a954490e959ce..15e3b14280e272 100644 --- a/paddle/pir/dialect/control_flow/ir/cf_type.h +++ b/paddle/pir/dialect/control_flow/ir/cf_type.h @@ -19,7 +19,14 @@ #include "paddle/pir/core/type_base.h" namespace pir { -class IR_API StackType : public Type::TypeBase { + +class IR_API ContainerType : public Type { + using Type::Type; + static bool classof(Type); +}; + +class IR_API StackType + : public Type::TypeBase { public: using Base::Base; }; @@ -36,6 +43,7 @@ class IR_API OutletType : public Type::TypeBase { } // namespace pir +IR_EXPORT_DECLARE_EXPLICIT_TYPE_ID(pir::ContainerType) IR_EXPORT_DECLARE_EXPLICIT_TYPE_ID(pir::StackType) IR_EXPORT_DECLARE_EXPLICIT_TYPE_ID(pir::InletType) IR_EXPORT_DECLARE_EXPLICIT_TYPE_ID(pir::OutletType) diff --git a/test/cpp/pir/control_flow_dialect/if_op_test.cc b/test/cpp/pir/control_flow_dialect/if_op_test.cc index 64ccbcdf270224..d078828029a414 100644 --- a/test/cpp/pir/control_flow_dialect/if_op_test.cc +++ b/test/cpp/pir/control_flow_dialect/if_op_test.cc @@ -117,8 +117,8 @@ TEST(if_op_test, network_with_backward) { auto x = builder.Build(std::vector{2, 2}, 1.0f).out(); auto y = builder.Build(std::vector{2, 2}, 2.0f).out(); auto cond = builder.Build(x, y).out(); - auto [stack_0, inlet_0, outlet_0] = builder.Build().out(); - auto [stack_1, inlet_1, outlet_1] = builder.Build().out(); + auto [stack_0, inlet_0, outlet_0] = builder.Build().out(); + auto [stack_1, inlet_1, outlet_1] = builder.Build().out(); (void)(stack_0); (void)(stack_1); @@ -127,15 +127,15 @@ TEST(if_op_test, network_with_backward) { builder.SetInsertionPointToStart(if_op.true_block()); auto local1_z = builder.Build(x, y).out(); auto local1_w = builder.Build(local1_z, y).out(); - builder.Build(inlet_0, - std::initializer_list{local1_z}); + builder.Build(inlet_0, + std::initializer_list{local1_z}); builder.Build(std::vector{local1_w}); builder.SetInsertionPointToStart(if_op.false_block()); auto local2_z = builder.Build(x, y).out(); auto local2_w = builder.Build(local2_z, y).out(); - builder.Build(inlet_1, - std::initializer_list{local2_z}); + builder.Build(inlet_1, + std::initializer_list{local2_z}); builder.Build(std::vector{local2_w}); builder.SetInsertionPointToEnd(block); @@ -148,7 +148,8 @@ TEST(if_op_test, network_with_backward) { // construct the true block of if_grad builder.SetInsertionPointToStart(if_grad.true_block()); - auto pop_local1_z = builder.Build(outlet_0).outlet_element(0); + auto pop_local1_z = + builder.Build(outlet_0).outlet_element(0); auto local1_add_grad_op = builder.Build(pop_local1_z, y, out_grad); auto pop_local1_z_grad = local1_add_grad_op.x_grad(), local1_y_grad_0 = local1_add_grad_op.y_grad(); @@ -162,7 +163,8 @@ TEST(if_op_test, network_with_backward) { // construct the false block of if_grad builder.SetInsertionPointToStart(if_grad.false_block()); - auto pop_local2_z = builder.Build(outlet_1).outlet_element(0); + auto pop_local2_z = + builder.Build(outlet_1).outlet_element(0); auto local2_matmul_grad_op = builder.Build(pop_local2_z, y, out_grad); auto pop_local2_z_grad = local2_matmul_grad_op.x_grad(), diff --git a/test/cpp/pir/control_flow_dialect/while_op_test.cc b/test/cpp/pir/control_flow_dialect/while_op_test.cc index d6a078a4ee9ce1..ed778ec8c311d8 100644 --- a/test/cpp/pir/control_flow_dialect/while_op_test.cc +++ b/test/cpp/pir/control_flow_dialect/while_op_test.cc @@ -98,7 +98,7 @@ TEST(while_op_test, network_with_backward) { // } auto cond_value = builder.Build(i, ten).out(); - auto [stack, inlet, outlet] = builder.Build().out(); + auto [stack, inlet, outlet] = builder.Build().out(); (void)(stack); auto while_op = builder.Build(cond_value, std::vector{i, x}); @@ -116,7 +116,7 @@ TEST(while_op_test, network_with_backward) { // comput new condition value: new_i < new_ten auto new_cond_value = builder.Build(new_i, ten).out(); - builder.Build( + builder.Build( inlet, std::initializer_list{body_x_argument}); builder.Build( @@ -146,7 +146,7 @@ TEST(while_op_test, network_with_backward) { auto local_x_out_grad_arg = bwd_body_block->AddArgument(x.type()); auto local_y_grad_arg = bwd_body_block->AddArgument(y.type()); - auto pop_op = builder.Build(outlet); + auto pop_op = builder.Build(outlet); auto bwd_body_x_argument = pop_op.outlet_element(0); auto add_grad_op = From 5547702a1fb8c412b0fe69c68ff36f32a974f959 Mon Sep 17 00:00:00 2001 From: WangZhen <23097963+0x45f@users.noreply.github.com> Date: Tue, 21 Nov 2023 11:05:22 +0800 Subject: [PATCH 39/46] Open yolo loss uts (#59154) --- test/legacy_test/test_yolov3_loss_op.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/legacy_test/test_yolov3_loss_op.py b/test/legacy_test/test_yolov3_loss_op.py index 552890e585323e..61984ffce1a56e 100644 --- a/test/legacy_test/test_yolov3_loss_op.py +++ b/test/legacy_test/test_yolov3_loss_op.py @@ -273,11 +273,13 @@ def setUp(self): def test_check_output(self): place = core.CPUPlace() - self.check_output_with_place(place, atol=2e-3) + self.check_output_with_place(place, atol=2e-3, check_pir=True) def test_check_grad_ignore_gtbox(self): place = core.CPUPlace() - self.check_grad_with_place(place, ['X'], 'Loss', max_relative_error=0.2) + self.check_grad_with_place( + place, ['X'], 'Loss', max_relative_error=0.2, check_pir=True + ) def initTestCase(self): self.anchors = [ From efdbd8b20cf8e545d269adb36d3b0df9e7b304a1 Mon Sep 17 00:00:00 2001 From: HongyuJia Date: Tue, 21 Nov 2023 11:13:09 +0800 Subject: [PATCH 40/46] [CINN] Strong constraint branch adapt to pir (#58993) * Strong Constraint Branch * NoInlineTranslator (#84) * Adapt adt to pir * Move FLAGS_cinn_enable_map_expr_schedule location * Apply new group schedule * Remove useless log * Remove adt unittest * Solve merge conflicts * Fix typo * Fix merge conflicts * Add unit test * Fix cmake * Add test_cinn_sub_graph_map_expr * Save current workspace * Remove unittest to cinn directory * Refactor unittest cmake * Refactor unittest cmake * Add unit test at Cmake * Restore test_cinn_sub_graph.py * Fix unittest * Refine codes according to comment * Refine codes according to comment --- paddle/cinn/adt/CMakeLists.txt | 62 ++++----- paddle/cinn/adt/adapter_tensor.cc | 44 +++++++ paddle/cinn/adt/adapter_tensor.h | 45 ++----- paddle/cinn/adt/equation_value.h | 5 - paddle/cinn/adt/generate_map_expr.cc | 118 +++++++++--------- paddle/cinn/adt/generate_map_expr.h | 22 ++-- paddle/cinn/adt/inline_translator.h | 37 +----- paddle/cinn/adt/inline_translator_trait.h | 58 +++++++++ paddle/cinn/adt/kgroup.cc | 4 +- paddle/cinn/adt/kgroup.h | 12 +- paddle/cinn/adt/m_expr.h | 23 ++-- paddle/cinn/adt/map_expr_ctx.h | 6 +- paddle/cinn/adt/naive_op_equation_context.cc | 45 ++++--- paddle/cinn/adt/naive_op_equation_context.h | 8 +- paddle/cinn/adt/no_inline_translator.h | 83 ++++++++++++ paddle/cinn/adt/print_utils/CMakeLists.txt | 27 ++-- .../cinn/adt/print_utils/print_equations.cc | 14 ++- paddle/cinn/adt/print_utils/print_map_expr.cc | 18 +-- paddle/cinn/adt/schedule_descriptor.cc | 4 +- .../transforms/cinn_group_lowering_pass.cc | 7 ++ paddle/cinn/hlir/framework/graph.h | 16 --- .../cinn/hlir/framework/op_lowering_impl.cc | 94 -------------- paddle/cinn/hlir/framework/op_lowering_impl.h | 38 ------ paddle/cinn/hlir/framework/pir/group.h | 31 ++++- .../hlir/framework/pir/op_lowering_impl.cc | 105 +++++++++++++++- .../hlir/framework/pir/op_lowering_impl.h | 33 +++++ paddle/cinn/hlir/pe/CMakeLists.txt | 7 +- paddle/cinn/hlir/pe/map_expr_to_ir.cc | 108 +++++++++++----- paddle/cinn/hlir/pe/map_expr_to_ir.h | 1 + .../st_shape_group_scheduler.cc | 7 ++ .../group_schedule/st_shape_group_scheduler.h | 2 + paddle/cinn/pybind/frontend.cc | 2 - paddle/cinn/runtime/flags.cc | 11 +- .../framework/paddle2cinn/cinn_compiler.cc | 2 - test/CMakeLists.txt | 2 +- test/cinn/CMakeLists.txt | 25 +--- test/cinn/adt/CMakeLists.txt | 21 ++++ test/cinn/adt/test_add_inline.py | 60 --------- test/cinn/adt/test_broadcast_expr.py | 59 --------- test/cinn/adt/test_cinn_sub_graph_map_expr.py | 76 +++++++++++ test/cinn/adt/test_naive_add.py | 57 --------- test/cinn/adt/test_naive_reduce.py | 50 -------- test/cinn/adt/test_reduce_fusion.py | 62 --------- test/cinn/adt/test_reduce_schedule_mesh.py | 59 --------- 44 files changed, 752 insertions(+), 818 deletions(-) create mode 100644 paddle/cinn/adt/adapter_tensor.cc create mode 100644 paddle/cinn/adt/inline_translator_trait.h create mode 100644 paddle/cinn/adt/no_inline_translator.h create mode 100644 test/cinn/adt/CMakeLists.txt delete mode 100755 test/cinn/adt/test_add_inline.py delete mode 100755 test/cinn/adt/test_broadcast_expr.py create mode 100644 test/cinn/adt/test_cinn_sub_graph_map_expr.py delete mode 100755 test/cinn/adt/test_naive_add.py delete mode 100644 test/cinn/adt/test_naive_reduce.py delete mode 100644 test/cinn/adt/test_reduce_fusion.py delete mode 100644 test/cinn/adt/test_reduce_schedule_mesh.py diff --git a/paddle/cinn/adt/CMakeLists.txt b/paddle/cinn/adt/CMakeLists.txt index 8dbb3bc8769839..e74c21bb0e7949 100644 --- a/paddle/cinn/adt/CMakeLists.txt +++ b/paddle/cinn/adt/CMakeLists.txt @@ -1,35 +1,39 @@ -add_subdirectory(print_utils) +if(NOT CINN_ONLY) + add_subdirectory(print_utils) -core_gather_headers() + core_gather_headers() -gather_srcs( - cinnapi_src - SRCS - anchor_sd_equation_context.cc - equation_function.cc - equation_solver.cc - equation_value.cc - generate_map_expr.cc - get_sub_reshape_dim_ranges.cc - igroup.cc - index_expr_infer_context.cc - kgroup.cc - m_ir.cc - naive_bidirection_equation_generator.cc - naive_op_equation_context.cc - partition_op_stmts.cc - schedule_descriptor.cc - schedule_dim.cc - schedule_mesh.cc - simplify_value.cc - write_broadcast_disabled_bidirection_equation_generator.cc) + gather_srcs( + cinnapi_src + SRCS + adapter_tensor.cc + anchor_sd_equation_context.cc + equation_function.cc + equation_solver.cc + equation_value.cc + generate_map_expr.cc + get_sub_reshape_dim_ranges.cc + igroup.cc + index_expr_infer_context.cc + kgroup.cc + m_ir.cc + naive_bidirection_equation_generator.cc + naive_op_equation_context.cc + partition_op_stmts.cc + schedule_descriptor.cc + schedule_dim.cc + schedule_mesh.cc + simplify_value.cc + write_broadcast_disabled_bidirection_equation_generator.cc) -cinn_cc_test(equation_value_match_trait_test SRCS - equation_value_match_trait_test.cc DEPS gtest glog) + cinn_cc_test(equation_value_match_trait_test SRCS + equation_value_match_trait_test.cc DEPS gtest glog) -cinn_cc_test(tree_test SRCS tree_test.cc DEPS gtest glog) + cinn_cc_test(tree_test SRCS tree_test.cc DEPS gtest glog) -cinn_cc_test(inline_translator_test SRCS inline_translator_test.cc DEPS - cinncore) + cinn_cc_test(inline_translator_test SRCS inline_translator_test.cc DEPS + cinncore) -message(STATUS "ADT srcs: ${cinnapi_src}") + message(STATUS "ADT srcs: ${cinnapi_src}") + +endif() diff --git a/paddle/cinn/adt/adapter_tensor.cc b/paddle/cinn/adt/adapter_tensor.cc new file mode 100644 index 00000000000000..464c45780dbecd --- /dev/null +++ b/paddle/cinn/adt/adapter_tensor.cc @@ -0,0 +1,44 @@ +// Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/cinn/adt/adapter_tensor.h" +#include "glog/logging.h" +#include "paddle/cinn/hlir/framework/pir/utils.h" + +namespace cinn::adt::adapter { + +std::size_t Tensor::GetRank() const { + return cinn::hlir::framework::pir::CompatibleInfo::ValueShape(node_data) + .size(); +} + +std::vector Tensor::GetShape() const { + std::vector ret{}; + for (int dim_size : + cinn::hlir::framework::pir::CompatibleInfo::ValueShape(node_data)) { + ret.emplace_back(dim_size); + } + return ret; +} + +std::size_t Tensor::GetNumel() const { + std::size_t ret = 1; + for (int dim_size : + cinn::hlir::framework::pir::CompatibleInfo::ValueShape(node_data)) { + ret = ret * dim_size; + } + return ret; +} + +} // namespace cinn::adt::adapter diff --git a/paddle/cinn/adt/adapter_tensor.h b/paddle/cinn/adt/adapter_tensor.h index 2a6cc941afb89e..dbd2c2dcecfdbb 100644 --- a/paddle/cinn/adt/adapter_tensor.h +++ b/paddle/cinn/adt/adapter_tensor.h @@ -13,59 +13,28 @@ // limitations under the License. #pragma once -#include "glog/logging.h" #include "paddle/cinn/adt/adt.h" -#include "paddle/cinn/hlir/framework/graph.h" -#include "paddle/cinn/hlir/framework/node.h" +#include "paddle/pir/core/value.h" namespace cinn::adt::adapter { struct Tensor final { - const hlir::framework::NodeData* node_data; - const hlir::framework::Graph* graph; + ::pir::Value node_data; bool operator==(const Tensor& other) const { - return this->node_data == other.node_data && this->graph == other.graph; + return this->node_data == other.node_data; } - std::size_t GetRank() const { - const auto& shape_dict = - graph->GetAttrs>( - "infershape"); - CHECK(shape_dict.count(node_data->id())) - << "Can't find " << node_data->id() << " 's shape!"; - return shape_dict.at(node_data->id()).size(); - } + std::size_t GetRank() const; - const std::vector& GetShape() const { - const auto& shape_dict = - graph->GetAttrs>( - "infershape"); - CHECK(shape_dict.count(node_data->id())) - << "Can't find " << node_data->id() << " 's shape!"; - return shape_dict.at(node_data->id()); - } + std::vector GetShape() const; - std::size_t GetNumel() const { - const auto& shape_dict = - graph->GetAttrs>( - "infershape"); - CHECK(shape_dict.count(node_data->id())) - << "Can't find " << node_data->id() << " 's shape!"; - std::vector shape = shape_dict.at(node_data->id()); - std::size_t ret = 1; - for (int32_t dim_size : shape) { - ret = ret * dim_size; - } - return ret; - } + std::size_t GetNumel() const; }; inline std::size_t GetHashValueImpl(const Tensor& tensor) { - return hash_combine( - std::hash()(tensor.node_data), - std::hash()(tensor.graph)); + return std::hash<::pir::Value>()(tensor.node_data); } } // namespace cinn::adt::adapter diff --git a/paddle/cinn/adt/equation_value.h b/paddle/cinn/adt/equation_value.h index 7aa6c2b7c3155b..6c1fef21a93dde 100644 --- a/paddle/cinn/adt/equation_value.h +++ b/paddle/cinn/adt/equation_value.h @@ -20,11 +20,6 @@ #include "paddle/cinn/adt/equation.h" #include "paddle/cinn/adt/match.h" -namespace cinn::hlir::framework { -class Node; -class NodeData; -} // namespace cinn::hlir::framework - namespace cinn::adt { DEFINE_ADT_TAG(tPointer); diff --git a/paddle/cinn/adt/generate_map_expr.cc b/paddle/cinn/adt/generate_map_expr.cc index b435acbcbcfb95..4180c9174a45fb 100644 --- a/paddle/cinn/adt/generate_map_expr.cc +++ b/paddle/cinn/adt/generate_map_expr.cc @@ -26,7 +26,11 @@ #include "paddle/cinn/adt/print.h" #include "paddle/cinn/adt/schedule_descriptor.h" #include "paddle/cinn/adt/tree.h" +#include "paddle/cinn/hlir/framework/pir/group.h" +#include "paddle/cinn/hlir/framework/pir/utils.h" #include "paddle/cinn/runtime/flags.h" +#include "paddle/pir/core/operation.h" +#include "paddle/pir/core/value.h" #include "glog/logging.h" @@ -84,79 +88,65 @@ using LoopDescriptor4IterVarT = std::function; using AnchorTensor = Variable; using FakeOpPlaceHolders = List; -Op MakeOp(const hlir::framework::Node* op) { return {op}; } +Op MakeOp(const ::pir::Operation* op) { return {op}; } template -void VisitEachInputTensor(const hlir::framework::Node* op, - const DoEachT& DoEach) { - for (const auto& graph_edge : op->inlinks_in_order()) { - DoEach(graph_edge->source()->safe_as()); +void VisitEachInputTensor(const ::pir::Operation* op, const DoEachT& DoEach) { + for (std::size_t i = 0; i < op->num_operands(); ++i) { + DoEach(op->operand_source(i)); } } -List MakeOpStmtInputList(const hlir::framework::Node* op, - const hlir::framework::Graph* graph) { +List MakeOpStmtInputList(const ::pir::Operation* op) { List ret{}; - VisitEachInputTensor(op, [&](const auto* tensor) { - ret->emplace_back(adapter::Tensor{tensor, graph}); + VisitEachInputTensor(op, [&](const ::pir::Value& tensor) { + ret->emplace_back(adapter::Tensor{tensor}); }); return ret; } template -void VisitEachOutputTensor(const hlir::framework::Node* op, - const DoEachT& DoEach) { - for (const auto& graph_edge : op->outlinks_in_order()) { - DoEach(graph_edge->sink()->safe_as()); +void VisitEachOutputTensor(const ::pir::Operation* op, const DoEachT& DoEach) { + for (std::size_t i = 0; i < op->num_results(); ++i) { + DoEach(const_cast<::pir::Operation*>(op)->result(i)); } } -List MakeOpStmtOutputList(const hlir::framework::Node* op, - const hlir::framework::Graph* graph) { +List MakeOpStmtOutputList(const ::pir::Operation* op) { List ret{}; - VisitEachOutputTensor(op, [&](const auto* tensor) { - ret->emplace_back(adapter::Tensor{tensor, graph}); + VisitEachOutputTensor(op, [&](const ::pir::Value& tensor) { + ret->emplace_back(adapter::Tensor{tensor}); }); return ret; } template -void VisitEachOpStmt( - const std::shared_ptr& group, - const DoEachT& DoEach) { - // Note - for (const auto* op : group->CollectNodes()) { - DoEach(OpStmt{MakeOp(op), - MakeOpStmtInputList(op, group->graph_), - MakeOpStmtOutputList(op, group->graph_)}); +void VisitEachOpStmt(const std::shared_ptr& group, + const DoEachT& DoEach) { + for (const auto* op : group->CollectOps()) { + DoEach( + OpStmt{MakeOp(op), MakeOpStmtInputList(op), MakeOpStmtOutputList(op)}); } } -hlir::framework::OpPatternKind GetOpPatternKind( - const hlir::framework::Node* node) { - static const hlir::framework::OpValueType& - op_pattern_dict = - hlir::framework::Operator::GetAttrs( - "OpPattern"); - auto kind = op_pattern_dict[node->op()]; - return kind; +hlir::framework::OpPatternKind GetOpPatternKind(const ::pir::Operation* node) { + return hlir::framework::pir::CompatibleInfo::OpKind(*node); } bool CollectRewritedReductionOpStmts(const OpStmt& op_stmt, List* ret) { const auto& [op, inputs, outputs] = op_stmt.tuple(); - CHECK(op.Has()); - if (GetOpPatternKind(op.Get()) == + CHECK(op.Has()); + if (GetOpPatternKind(op.Get()) == hlir::framework::OpPatternKind::kReduction) { - tReduceInit init_op{ - op.Get()}; + tReduceInit init_op{ + op.Get()}; (*ret)->emplace_back(OpStmt{init_op, List{}, outputs}); - tReduceAcc acc_op{ - op.Get()}; + tReduceAcc acc_op{op.Get()}; (*ret)->emplace_back(OpStmt{acc_op, inputs, outputs}); return true; } else { @@ -172,7 +162,7 @@ void CollectRewritedOpStmts(const OpStmt& op_stmt, List* ret) { } List MakeOpStmts( - const std::shared_ptr& group) { + const std::shared_ptr& group) { List ret{}; VisitEachOpStmt(group, [&](const auto& op_stmt) { @@ -213,7 +203,7 @@ std::shared_ptr MakeIGroup(const AnchorGroup& igroup_spec) { } std::vector> GenerateIGroups( - const std::shared_ptr& group) { + const std::shared_ptr& group) { std::vector> ret{}; List op_stmts = MakeOpStmts(group); @@ -227,7 +217,7 @@ std::vector> GenerateIGroups( } std::shared_ptr GenerateKGroups( - const std::shared_ptr& group, + const std::shared_ptr& group, const std::vector>& igroups) { CHECK_EQ(igroups.size(), 1); return std::make_shared(group, igroups); @@ -343,36 +333,34 @@ Tensor GetAnchorTensor(const std::shared_ptr& igroup) { } template -void VisitInputTensor(const hlir::framework::Graph::Group& group, +void VisitInputTensor(const hlir::framework::pir::Group& group, const DoEachT& DoEach) { - for (const auto* node_data : group.GetInputNodeDatas()) { - DoEach(node_data, group.graph_); + for (const ::pir::Value& node_data : group.GetInputOpValues()) { + DoEach(node_data); } } template -void VisitOutputTensor(const hlir::framework::Graph::Group& group, +void VisitOutputTensor(const hlir::framework::pir::Group& group, const DoEachT& DoEach) { - for (const auto& node_data : group.GetOutputNodeDatas()) { - DoEach(node_data, group.graph_); + for (const ::pir::Value& node_data : group.GetOutputOpValues()) { + DoEach(node_data); } } List MakeInputTensors(const std::shared_ptr& kgroup) { List ret{}; - VisitInputTensor(*kgroup->cinn_group(), - [&](const auto* node_data, const auto* graph) { - ret->emplace_back(adapter::Tensor{node_data, graph}); - }); + VisitInputTensor(*kgroup->cinn_group(), [&](const ::pir::Value& node_data) { + ret->emplace_back(adapter::Tensor{node_data}); + }); return ret; } List MakeOutputTensors(const std::shared_ptr& kgroup) { List ret{}; - VisitOutputTensor(*kgroup->cinn_group(), - [&](const auto* node_data, const auto* graph) { - ret->emplace_back(adapter::Tensor{node_data, graph}); - }); + VisitOutputTensor(*kgroup->cinn_group(), [&](const ::pir::Value& node_data) { + ret->emplace_back(adapter::Tensor{node_data}); + }); return ret; } @@ -437,7 +425,7 @@ MapExpr GenerateMapExpr(const std::shared_ptr& kgroup) { } // namespace MapExpr GenerateMapExpr( - const std::shared_ptr& group) { + const std::shared_ptr& group) { const auto& igroups = GenerateIGroups(group); const auto& kgroup = GenerateKGroups(group, igroups); @@ -445,18 +433,26 @@ MapExpr GenerateMapExpr( return GenerateMapExpr(kgroup); } -namespace {} // namespace - void TryGenerateMapExprFromGraph( - const std::shared_ptr& graph) { + const hlir::framework::pir::GroupList& groups) { if (!FLAGS_cinn_enable_map_expr) { return; } - for (const auto& fusion_group : graph->fusion_groups) { + for (const auto& fusion_group : groups) { const auto& map_expr = GenerateMapExpr(fusion_group); VLOG(1) << ToTxtString(map_expr, fusion_group->group_id); fusion_group->set_map_expr_ctx(std::make_shared(map_expr)); } } +void TryGenerateMapExprFromGroup( + const std::shared_ptr& fusion_group) { + if (!FLAGS_cinn_enable_map_expr) { + return; + } + const auto& map_expr = GenerateMapExpr(fusion_group); + VLOG(1) << ToTxtString(map_expr, fusion_group->group_id); + fusion_group->set_map_expr_ctx(std::make_shared(map_expr)); +} + } // namespace cinn::adt diff --git a/paddle/cinn/adt/generate_map_expr.h b/paddle/cinn/adt/generate_map_expr.h index c604dd9f070c06..61b5906c8138a3 100644 --- a/paddle/cinn/adt/generate_map_expr.h +++ b/paddle/cinn/adt/generate_map_expr.h @@ -14,19 +14,25 @@ #pragma once +#include + #include "paddle/cinn/adt/m_expr.h" -#include "paddle/cinn/adt/m_ir.h" -#include "paddle/cinn/hlir/framework/graph.h" -namespace cinn::adt { +namespace cinn::hlir::framework::pir { + +struct Group; +using GroupList = std::vector>; -class IGroup; -class KGroup; +} // namespace cinn::hlir::framework::pir + +namespace cinn::adt { MapExpr GenerateMapExpr( - const std::shared_ptr& group); + const std::shared_ptr& group); + +void TryGenerateMapExprFromGraph(const hlir::framework::pir::GroupList& groups); -void TryGenerateMapExprFromGraph( - const std::shared_ptr& graph); +void TryGenerateMapExprFromGroup( + const std::shared_ptr& fusion_group); } // namespace cinn::adt diff --git a/paddle/cinn/adt/inline_translator.h b/paddle/cinn/adt/inline_translator.h index 5298d17ffadcda..2cd3a44bc7dd05 100644 --- a/paddle/cinn/adt/inline_translator.h +++ b/paddle/cinn/adt/inline_translator.h @@ -15,47 +15,12 @@ #pragma once #include "paddle/cinn/adt/adt.h" +#include "paddle/cinn/adt/inline_translator_trait.h" #include "paddle/cinn/adt/m_expr.h" #include "paddle/cinn/adt/tree.h" namespace cinn::adt { -template