From 94aa1a39a5f698519c7abb651c66a563a6810086 Mon Sep 17 00:00:00 2001 From: Eddie-Wang1120 Date: Sat, 16 Mar 2024 10:10:35 +0000 Subject: [PATCH 1/4] fix test_fused_adam_op --- paddle/fluid/pir/dialect/op_generator/ops_api_gen.py | 1 + paddle/fluid/pir/dialect/operator/ir/ops.yaml | 5 ++--- paddle/phi/api/yaml/legacy_ops.yaml | 5 ++--- paddle/phi/api/yaml/op_compat.yaml | 9 +++++++++ test/white_list/pir_op_test_white_list | 1 + 5 files changed, 15 insertions(+), 6 deletions(-) diff --git a/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py b/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py index 6a86600c1f5f2..3ccd6218397e0 100644 --- a/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py +++ b/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py @@ -143,6 +143,7 @@ 'dpsgd', 'embedding_grad_sparse', 'ftrl', + 'fused_adam', 'fused_batch_norm_act_', 'fused_bn_add_activation_', 'fused_elemwise_add_activation', diff --git a/paddle/fluid/pir/dialect/operator/ir/ops.yaml b/paddle/fluid/pir/dialect/operator/ir/ops.yaml index 60f1bf2f03941..ea7496dfadc07 100644 --- a/paddle/fluid/pir/dialect/operator/ir/ops.yaml +++ b/paddle/fluid/pir/dialect/operator/ir/ops.yaml @@ -741,7 +741,7 @@ data_type : dtype interfaces : paddle::dialect::InferSymbolicShapeInterface -- op : fused_adam_ +- op : fused_adam args : (Tensor[] params, Tensor[] grads, Tensor learning_rate, Tensor[] moments1, Tensor[] moments2, Tensor[] beta1_pows, Tensor[] beta2_pows, Tensor[] master_params, Tensor skip_update, Scalar beta1, Scalar beta2, Scalar epsilon, int chunk_size, float weight_decay, bool use_adamw, bool multi_precision, bool use_global_beta_pow) output : Tensor[](params_out){params.size()}, Tensor[](moments1_out){params.size()}, Tensor[](moments2_out){params.size()}, Tensor[](beta1_pows_out){params.size()}, Tensor[](beta2_pows_out){params.size()}, Tensor[](master_params_out){params.size()} infer_meta : @@ -749,8 +749,7 @@ kernel : func : fused_adam data_type : params - optional : skip_update, master_params - inplace : (params -> params_out), (moments1 -> moments1_out), (moments2 -> moments2_out), (beta1_pows -> beta1_pows_out), (beta2_pows -> beta2_pows_out), (master_params -> master_params_out) + optional : skip_update, master_params, master_params_out - op : fused_batch_norm_act args : (Tensor x, Tensor scale, Tensor bias, Tensor mean, Tensor variance, float momentum, float epsilon, str act_type) diff --git a/paddle/phi/api/yaml/legacy_ops.yaml b/paddle/phi/api/yaml/legacy_ops.yaml index 142814e1cc01e..f8e4f8b90ee2e 100755 --- a/paddle/phi/api/yaml/legacy_ops.yaml +++ b/paddle/phi/api/yaml/legacy_ops.yaml @@ -560,7 +560,7 @@ func : full_with_tensor data_type : dtype -- op : fused_adam_ +- op : fused_adam args : (Tensor[] params, Tensor[] grads, Tensor learning_rate, Tensor[] moments1, Tensor[] moments2, Tensor[] beta1_pows, Tensor[] beta2_pows, Tensor[] master_params, Tensor skip_update, Scalar beta1, Scalar beta2, Scalar epsilon, int chunk_size, float weight_decay, bool use_adamw, bool multi_precision, bool use_global_beta_pow) output : Tensor[](params_out){params.size()}, Tensor[](moments1_out){params.size()}, Tensor[](moments2_out){params.size()}, Tensor[](beta1_pows_out){params.size()}, Tensor[](beta2_pows_out){params.size()}, Tensor[](master_params_out){params.size()} infer_meta : @@ -568,8 +568,7 @@ kernel : func : fused_adam data_type : params - optional : skip_update, master_params - inplace : (params -> params_out), (moments1 -> moments1_out), (moments2 -> moments2_out), (beta1_pows -> beta1_pows_out), (beta2_pows -> beta2_pows_out), (master_params -> master_params_out) + optional : skip_update, master_params, master_params_out - op : fused_batch_norm_act args : (Tensor x, Tensor scale, Tensor bias, Tensor mean, Tensor variance, float momentum, float epsilon, str act_type) diff --git a/paddle/phi/api/yaml/op_compat.yaml b/paddle/phi/api/yaml/op_compat.yaml index 2fbd73623f0a0..a54046166feda 100755 --- a/paddle/phi/api/yaml/op_compat.yaml +++ b/paddle/phi/api/yaml/op_compat.yaml @@ -1254,6 +1254,15 @@ data_type : float support_tensor : true +- op : fused_adam + inputs : + {params : Params, grads : Grads, learning_rate : LearningRate, moments1 : Moments1, + moments2 : Moments2, beta1_pows : Beta1Pows, beta2_pows : Beta2Pows, master_params : MasterParams, + skip_update : SkipUpdate} + outputs : + {params_out : ParamsOut, moments1_out : Moments1Out, moments2_out : Moments2Out, + beta1_pows_out : Beta1PowsOut, beta2_pows_out : Beta2PowsOut, master_params_out : MasterParamsOut} + - op : fused_attention backward: fused_attention_grad inputs: diff --git a/test/white_list/pir_op_test_white_list b/test/white_list/pir_op_test_white_list index dfa901c0ca126..c9bac8c330e16 100644 --- a/test/white_list/pir_op_test_white_list +++ b/test/white_list/pir_op_test_white_list @@ -109,6 +109,7 @@ test_fold_op test_frame_op test_ftrl_op test_full_like_op +test_fused_adam_op test_fused_attention_op test_fused_attention_op_api test_fused_bias_dropout_residual_layer_norm_op From 75081c27f1180d2a3df7bd1dab12015db7d85b0c Mon Sep 17 00:00:00 2001 From: Eddie-Wang1120 Date: Mon, 18 Mar 2024 12:18:13 +0000 Subject: [PATCH 2/4] show error --- paddle/fluid/pir/dialect/operator/ir/ops.yaml | 12 ++++++++++++ paddle/phi/api/yaml/legacy_ops.yaml | 12 ++++++++++++ 2 files changed, 24 insertions(+) diff --git a/paddle/fluid/pir/dialect/operator/ir/ops.yaml b/paddle/fluid/pir/dialect/operator/ir/ops.yaml index ea7496dfadc07..2975e4b4b076e 100644 --- a/paddle/fluid/pir/dialect/operator/ir/ops.yaml +++ b/paddle/fluid/pir/dialect/operator/ir/ops.yaml @@ -749,8 +749,20 @@ kernel : func : fused_adam data_type : params + inplace : (params -> params_out), (moments1 -> moments1_out), (moments2 -> moments2_out), (beta1_pows -> beta1_pows_out), (beta2_pows -> beta2_pows_out), (master_params -> master_params_out) optional : skip_update, master_params, master_params_out +- op : fused_adam_ + args : (Tensor[] params, Tensor[] grads, Tensor learning_rate, Tensor[] moments1, Tensor[] moments2, Tensor[] beta1_pows, Tensor[] beta2_pows, Tensor[] master_params, Tensor skip_update, Scalar beta1, Scalar beta2, Scalar epsilon, int chunk_size, float weight_decay, bool use_adamw, bool multi_precision, bool use_global_beta_pow) + output : Tensor[](params_out){params.size()}, Tensor[](moments1_out){params.size()}, Tensor[](moments2_out){params.size()}, Tensor[](beta1_pows_out){params.size()}, Tensor[](beta2_pows_out){params.size()}, Tensor[](master_params_out){params.size()} + infer_meta : + func : FusedAdamInferMeta + kernel : + func : fused_adam + data_type : params + optional : skip_update, master_params + inplace : (params -> params_out), (moments1 -> moments1_out), (moments2 -> moments2_out), (beta1_pows -> beta1_pows_out), (beta2_pows -> beta2_pows_out), (master_params -> master_params_out) + - op : fused_batch_norm_act args : (Tensor x, Tensor scale, Tensor bias, Tensor mean, Tensor variance, float momentum, float epsilon, str act_type) output : Tensor(out), Tensor(mean_out), Tensor(variance_out), Tensor(saved_mean), Tensor(saved_variance), Tensor(reserve_space) diff --git a/paddle/phi/api/yaml/legacy_ops.yaml b/paddle/phi/api/yaml/legacy_ops.yaml index f8e4f8b90ee2e..68375664949b9 100755 --- a/paddle/phi/api/yaml/legacy_ops.yaml +++ b/paddle/phi/api/yaml/legacy_ops.yaml @@ -568,8 +568,20 @@ kernel : func : fused_adam data_type : params + inplace : (params -> params_out), (moments1 -> moments1_out), (moments2 -> moments2_out), (beta1_pows -> beta1_pows_out), (beta2_pows -> beta2_pows_out), (master_params -> master_params_out) optional : skip_update, master_params, master_params_out +- op : fused_adam_ + args : (Tensor[] params, Tensor[] grads, Tensor learning_rate, Tensor[] moments1, Tensor[] moments2, Tensor[] beta1_pows, Tensor[] beta2_pows, Tensor[] master_params, Tensor skip_update, Scalar beta1, Scalar beta2, Scalar epsilon, int chunk_size, float weight_decay, bool use_adamw, bool multi_precision, bool use_global_beta_pow) + output : Tensor[](params_out){params.size()}, Tensor[](moments1_out){params.size()}, Tensor[](moments2_out){params.size()}, Tensor[](beta1_pows_out){params.size()}, Tensor[](beta2_pows_out){params.size()}, Tensor[](master_params_out){params.size()} + infer_meta : + func : FusedAdamInferMeta + kernel : + func : fused_adam + data_type : params + optional : skip_update, master_params + inplace : (params -> params_out), (moments1 -> moments1_out), (moments2 -> moments2_out), (beta1_pows -> beta1_pows_out), (beta2_pows -> beta2_pows_out), (master_params -> master_params_out) + - op : fused_batch_norm_act args : (Tensor x, Tensor scale, Tensor bias, Tensor mean, Tensor variance, float momentum, float epsilon, str act_type) output : Tensor(out), Tensor(mean_out), Tensor(variance_out), Tensor(saved_mean), Tensor(saved_variance), Tensor(reserve_space) From 7e411545e2bd1f72fd4fa9bc863a602cf2d0568d Mon Sep 17 00:00:00 2001 From: Eddie-Wang1120 Date: Tue, 19 Mar 2024 10:16:11 +0000 Subject: [PATCH 3/4] update fix --- .../fluid/pir/dialect/op_generator/ops_api_gen.py | 2 +- paddle/fluid/pir/dialect/operator/ir/ops.yaml | 13 +------------ paddle/phi/api/yaml/legacy_ops.yaml | 13 +------------ paddle/phi/api/yaml/op_compat.yaml | 2 +- 4 files changed, 4 insertions(+), 26 deletions(-) diff --git a/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py b/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py index 3ccd6218397e0..ab12929d1b813 100644 --- a/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py +++ b/paddle/fluid/pir/dialect/op_generator/ops_api_gen.py @@ -143,7 +143,7 @@ 'dpsgd', 'embedding_grad_sparse', 'ftrl', - 'fused_adam', + 'fused_adam_', 'fused_batch_norm_act_', 'fused_bn_add_activation_', 'fused_elemwise_add_activation', diff --git a/paddle/fluid/pir/dialect/operator/ir/ops.yaml b/paddle/fluid/pir/dialect/operator/ir/ops.yaml index 2975e4b4b076e..3708960f82263 100644 --- a/paddle/fluid/pir/dialect/operator/ir/ops.yaml +++ b/paddle/fluid/pir/dialect/operator/ir/ops.yaml @@ -741,17 +741,6 @@ data_type : dtype interfaces : paddle::dialect::InferSymbolicShapeInterface -- op : fused_adam - args : (Tensor[] params, Tensor[] grads, Tensor learning_rate, Tensor[] moments1, Tensor[] moments2, Tensor[] beta1_pows, Tensor[] beta2_pows, Tensor[] master_params, Tensor skip_update, Scalar beta1, Scalar beta2, Scalar epsilon, int chunk_size, float weight_decay, bool use_adamw, bool multi_precision, bool use_global_beta_pow) - output : Tensor[](params_out){params.size()}, Tensor[](moments1_out){params.size()}, Tensor[](moments2_out){params.size()}, Tensor[](beta1_pows_out){params.size()}, Tensor[](beta2_pows_out){params.size()}, Tensor[](master_params_out){params.size()} - infer_meta : - func : FusedAdamInferMeta - kernel : - func : fused_adam - data_type : params - inplace : (params -> params_out), (moments1 -> moments1_out), (moments2 -> moments2_out), (beta1_pows -> beta1_pows_out), (beta2_pows -> beta2_pows_out), (master_params -> master_params_out) - optional : skip_update, master_params, master_params_out - - op : fused_adam_ args : (Tensor[] params, Tensor[] grads, Tensor learning_rate, Tensor[] moments1, Tensor[] moments2, Tensor[] beta1_pows, Tensor[] beta2_pows, Tensor[] master_params, Tensor skip_update, Scalar beta1, Scalar beta2, Scalar epsilon, int chunk_size, float weight_decay, bool use_adamw, bool multi_precision, bool use_global_beta_pow) output : Tensor[](params_out){params.size()}, Tensor[](moments1_out){params.size()}, Tensor[](moments2_out){params.size()}, Tensor[](beta1_pows_out){params.size()}, Tensor[](beta2_pows_out){params.size()}, Tensor[](master_params_out){params.size()} @@ -760,7 +749,7 @@ kernel : func : fused_adam data_type : params - optional : skip_update, master_params + optional : skip_update, master_params, master_params_out inplace : (params -> params_out), (moments1 -> moments1_out), (moments2 -> moments2_out), (beta1_pows -> beta1_pows_out), (beta2_pows -> beta2_pows_out), (master_params -> master_params_out) - op : fused_batch_norm_act diff --git a/paddle/phi/api/yaml/legacy_ops.yaml b/paddle/phi/api/yaml/legacy_ops.yaml index 68375664949b9..7f9c7ec50c490 100755 --- a/paddle/phi/api/yaml/legacy_ops.yaml +++ b/paddle/phi/api/yaml/legacy_ops.yaml @@ -560,17 +560,6 @@ func : full_with_tensor data_type : dtype -- op : fused_adam - args : (Tensor[] params, Tensor[] grads, Tensor learning_rate, Tensor[] moments1, Tensor[] moments2, Tensor[] beta1_pows, Tensor[] beta2_pows, Tensor[] master_params, Tensor skip_update, Scalar beta1, Scalar beta2, Scalar epsilon, int chunk_size, float weight_decay, bool use_adamw, bool multi_precision, bool use_global_beta_pow) - output : Tensor[](params_out){params.size()}, Tensor[](moments1_out){params.size()}, Tensor[](moments2_out){params.size()}, Tensor[](beta1_pows_out){params.size()}, Tensor[](beta2_pows_out){params.size()}, Tensor[](master_params_out){params.size()} - infer_meta : - func : FusedAdamInferMeta - kernel : - func : fused_adam - data_type : params - inplace : (params -> params_out), (moments1 -> moments1_out), (moments2 -> moments2_out), (beta1_pows -> beta1_pows_out), (beta2_pows -> beta2_pows_out), (master_params -> master_params_out) - optional : skip_update, master_params, master_params_out - - op : fused_adam_ args : (Tensor[] params, Tensor[] grads, Tensor learning_rate, Tensor[] moments1, Tensor[] moments2, Tensor[] beta1_pows, Tensor[] beta2_pows, Tensor[] master_params, Tensor skip_update, Scalar beta1, Scalar beta2, Scalar epsilon, int chunk_size, float weight_decay, bool use_adamw, bool multi_precision, bool use_global_beta_pow) output : Tensor[](params_out){params.size()}, Tensor[](moments1_out){params.size()}, Tensor[](moments2_out){params.size()}, Tensor[](beta1_pows_out){params.size()}, Tensor[](beta2_pows_out){params.size()}, Tensor[](master_params_out){params.size()} @@ -579,7 +568,7 @@ kernel : func : fused_adam data_type : params - optional : skip_update, master_params + optional : skip_update, master_params, master_params_out inplace : (params -> params_out), (moments1 -> moments1_out), (moments2 -> moments2_out), (beta1_pows -> beta1_pows_out), (beta2_pows -> beta2_pows_out), (master_params -> master_params_out) - op : fused_batch_norm_act diff --git a/paddle/phi/api/yaml/op_compat.yaml b/paddle/phi/api/yaml/op_compat.yaml index a54046166feda..ead838959e39a 100755 --- a/paddle/phi/api/yaml/op_compat.yaml +++ b/paddle/phi/api/yaml/op_compat.yaml @@ -1254,7 +1254,7 @@ data_type : float support_tensor : true -- op : fused_adam +- op : fused_adam_(fused_adam) inputs : {params : Params, grads : Grads, learning_rate : LearningRate, moments1 : Moments1, moments2 : Moments2, beta1_pows : Beta1Pows, beta2_pows : Beta2Pows, master_params : MasterParams, From c71833dcfcd08ce17557a7df9e3e823d335ff833 Mon Sep 17 00:00:00 2001 From: Eddie-Wang1120 Date: Tue, 19 Mar 2024 13:24:40 +0000 Subject: [PATCH 4/4] recover legacy --- paddle/phi/api/yaml/legacy_ops.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/paddle/phi/api/yaml/legacy_ops.yaml b/paddle/phi/api/yaml/legacy_ops.yaml index 7f9c7ec50c490..142814e1cc01e 100755 --- a/paddle/phi/api/yaml/legacy_ops.yaml +++ b/paddle/phi/api/yaml/legacy_ops.yaml @@ -568,7 +568,7 @@ kernel : func : fused_adam data_type : params - optional : skip_update, master_params, master_params_out + optional : skip_update, master_params inplace : (params -> params_out), (moments1 -> moments1_out), (moments2 -> moments2_out), (beta1_pows -> beta1_pows_out), (beta2_pows -> beta2_pows_out), (master_params -> master_params_out) - op : fused_batch_norm_act