From 83c86118d669e25b6436bdb024003ce24163729b Mon Sep 17 00:00:00 2001 From: Xiaoxu Chen Date: Thu, 8 Sep 2022 09:29:38 +0000 Subject: [PATCH 1/6] add reduce_mean,reduce_sum primitive ops --- python/paddle/fluid/tests/unittests/autograd/test_primapi.py | 2 ++ python/paddle/incubate/autograd/primops.py | 2 ++ 2 files changed, 4 insertions(+) diff --git a/python/paddle/fluid/tests/unittests/autograd/test_primapi.py b/python/paddle/fluid/tests/unittests/autograd/test_primapi.py index d010e69e75950..ff83fd933a1f7 100644 --- a/python/paddle/fluid/tests/unittests/autograd/test_primapi.py +++ b/python/paddle/fluid/tests/unittests/autograd/test_primapi.py @@ -301,6 +301,8 @@ def test_illegal_param(self): ('abs', paddle.abs, (np.random.uniform(-10, 10, (200, 345)), ), None, 'float32'), )) + + class TestGrad(unittest.TestCase): def setUp(self): diff --git a/python/paddle/incubate/autograd/primops.py b/python/paddle/incubate/autograd/primops.py index dde3fb492cd97..1c71840f4ebe9 100644 --- a/python/paddle/incubate/autograd/primops.py +++ b/python/paddle/incubate/autograd/primops.py @@ -207,6 +207,8 @@ def reduce_sum(x, axis=None, keepdim=False, out=None): if not isinstance(keepdim, bool): raise TypeError(f'keepdim must be bool, but got {type(keepdim)}') + + attrs = {'axis': axis, 'keepdim': keepdim} helper = LayerHelper('reduce_sum_p', **locals()) if out is None: From 7bb38ec71528dddf95730a13bf726b24da52ee37 Mon Sep 17 00:00:00 2001 From: Xiaoxu Chen Date: Thu, 8 Sep 2022 10:50:45 +0000 Subject: [PATCH 2/6] add ne_p gt_p primitive operators --- .../tests/unittests/autograd/test_jvp_and_transpose.py | 2 -- .../fluid/tests/unittests/autograd/test_primapi.py | 1 - python/paddle/incubate/autograd/primops.py | 1 + python/paddle/incubate/autograd/primrules.py | 9 +++++++++ 4 files changed, 10 insertions(+), 3 deletions(-) diff --git a/python/paddle/fluid/tests/unittests/autograd/test_jvp_and_transpose.py b/python/paddle/fluid/tests/unittests/autograd/test_jvp_and_transpose.py index 76698a7a8b5fd..80d5ec8b0c42f 100644 --- a/python/paddle/fluid/tests/unittests/autograd/test_jvp_and_transpose.py +++ b/python/paddle/fluid/tests/unittests/autograd/test_jvp_and_transpose.py @@ -1040,7 +1040,6 @@ def init_data(self): # transpose op: ] - class TestGePJVPAndTranspose(TestAddPJVPAndTranspose): def init_data(self): @@ -1070,7 +1069,6 @@ def init_data(self): # transpose op: ] - class TestNePJVPAndTranspose(TestAddPJVPAndTranspose): def init_data(self): diff --git a/python/paddle/fluid/tests/unittests/autograd/test_primapi.py b/python/paddle/fluid/tests/unittests/autograd/test_primapi.py index ff83fd933a1f7..f518685108ab6 100644 --- a/python/paddle/fluid/tests/unittests/autograd/test_primapi.py +++ b/python/paddle/fluid/tests/unittests/autograd/test_primapi.py @@ -302,7 +302,6 @@ def test_illegal_param(self): (200, 345)), ), None, 'float32'), )) - class TestGrad(unittest.TestCase): def setUp(self): diff --git a/python/paddle/incubate/autograd/primops.py b/python/paddle/incubate/autograd/primops.py index 1c71840f4ebe9..6a84405e3e70f 100644 --- a/python/paddle/incubate/autograd/primops.py +++ b/python/paddle/incubate/autograd/primops.py @@ -366,6 +366,7 @@ def ge(x, y, out=None): return _simple_binop(LayerHelper('ge_p', **locals())) + @REGISTER_FN('ne_p', 'X', 'Y', 'Z') def ne(x, y, out=None): return _simple_binop(LayerHelper('ne_p', **locals())) diff --git a/python/paddle/incubate/autograd/primrules.py b/python/paddle/incubate/autograd/primrules.py index 954bdf0cb1c86..1a865af812017 100644 --- a/python/paddle/incubate/autograd/primrules.py +++ b/python/paddle/incubate/autograd/primrules.py @@ -349,6 +349,7 @@ def gt_orig2prim(op, x, y): return gt(x, y) +<<<<<<< 7036dbc3b81d1b4fffdf106835abdb2f40ee09d2 @REGISTER_ORIG2PRIM('greater_equal') def ge_orig2prim(op, x, y): if x.shape != y.shape: @@ -356,6 +357,8 @@ def ge_orig2prim(op, x, y): return ge(x, y) +======= +>>>>>>> add ne_p gt_p primitive operators @REGISTER_ORIG2PRIM('elementwise_pow') def elementwise_pow_orig2prim(op, x, y): if x.shape != y.shape: @@ -572,11 +575,14 @@ def gt_prim2orig(op, x, y): return paddle.greater_than(x, y) +<<<<<<< 7036dbc3b81d1b4fffdf106835abdb2f40ee09d2 @REGISTER_PRIM2ORIG('ge_p') def ge_prim2orig(op, x, y): return paddle.greater_equal(x, y) +======= +>>>>>>> add ne_p gt_p primitive operators @REGISTER_PRIM2ORIG('ne_p') def ne_prim2orig(op, x, y): return paddle.not_equal(x, y) @@ -863,6 +869,7 @@ def gt_jvp(op, x_dot, y_dot): return z_dot +<<<<<<< 7036dbc3b81d1b4fffdf106835abdb2f40ee09d2 @REGISTER_JVP('ge_p') def ge_jvp(op, x_dot, y_dot): if x_dot is None and y_dot is None: @@ -872,6 +879,8 @@ def ge_jvp(op, x_dot, y_dot): return z_dot +======= +>>>>>>> add ne_p gt_p primitive operators @REGISTER_JVP('ne_p') def ne_jvp(op, x_dot, y_dot): if x_dot is None and y_dot is None: From de97bd4c2ace4c472e3ba99f83bc627284af22f0 Mon Sep 17 00:00:00 2001 From: Xiaoxu Chen Date: Thu, 8 Sep 2022 11:45:11 +0000 Subject: [PATCH 3/6] add ge_p abs_p primitive oparators --- .../autograd/test_jvp_and_transpose.py | 30 +++++++++++++++++++ python/paddle/incubate/autograd/primops.py | 3 -- python/paddle/incubate/autograd/primrules.py | 11 ++----- 3 files changed, 32 insertions(+), 12 deletions(-) diff --git a/python/paddle/fluid/tests/unittests/autograd/test_jvp_and_transpose.py b/python/paddle/fluid/tests/unittests/autograd/test_jvp_and_transpose.py index 80d5ec8b0c42f..6ba8cce25782b 100644 --- a/python/paddle/fluid/tests/unittests/autograd/test_jvp_and_transpose.py +++ b/python/paddle/fluid/tests/unittests/autograd/test_jvp_and_transpose.py @@ -1069,6 +1069,36 @@ def init_data(self): # transpose op: ] +class TestGePJVPAndTranspose(TestAddPJVPAndTranspose): + + def init_data(self): + # Set prim op + self.op_type = 'ge_p' + X = paddle.static.data(name='X', shape=[4, 5], dtype='float64') + Y = paddle.static.data(name='Y', shape=[4, 5], dtype='float64') + + self.prim_input = {'X': X, 'Y': Y} + self.prim_output = { + 'Z': + self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + } + self.prim_attrs = {} + + # Set JVP + X_DOT = paddle.static.data(name='X_DOT', shape=[4, 5], dtype='float64') + Y_DOT = paddle.static.data(name='Y_DOT', shape=[4, 5], dtype='float64') + self.jvp_args = (X_DOT, Y_DOT) + self.jvp_out_shape_map = {0: self.prim_output['Z']} + + self.all_ops = [ + # prim op: + 'ge_p', + # jvp op: + 'fill_constant_p', + # transpose op: + ] + + class TestNePJVPAndTranspose(TestAddPJVPAndTranspose): def init_data(self): diff --git a/python/paddle/incubate/autograd/primops.py b/python/paddle/incubate/autograd/primops.py index 6a84405e3e70f..dde3fb492cd97 100644 --- a/python/paddle/incubate/autograd/primops.py +++ b/python/paddle/incubate/autograd/primops.py @@ -207,8 +207,6 @@ def reduce_sum(x, axis=None, keepdim=False, out=None): if not isinstance(keepdim, bool): raise TypeError(f'keepdim must be bool, but got {type(keepdim)}') - - attrs = {'axis': axis, 'keepdim': keepdim} helper = LayerHelper('reduce_sum_p', **locals()) if out is None: @@ -366,7 +364,6 @@ def ge(x, y, out=None): return _simple_binop(LayerHelper('ge_p', **locals())) - @REGISTER_FN('ne_p', 'X', 'Y', 'Z') def ne(x, y, out=None): return _simple_binop(LayerHelper('ne_p', **locals())) diff --git a/python/paddle/incubate/autograd/primrules.py b/python/paddle/incubate/autograd/primrules.py index 1a865af812017..6b09bb51e7a7c 100644 --- a/python/paddle/incubate/autograd/primrules.py +++ b/python/paddle/incubate/autograd/primrules.py @@ -349,7 +349,7 @@ def gt_orig2prim(op, x, y): return gt(x, y) -<<<<<<< 7036dbc3b81d1b4fffdf106835abdb2f40ee09d2 + @REGISTER_ORIG2PRIM('greater_equal') def ge_orig2prim(op, x, y): if x.shape != y.shape: @@ -357,8 +357,6 @@ def ge_orig2prim(op, x, y): return ge(x, y) -======= ->>>>>>> add ne_p gt_p primitive operators @REGISTER_ORIG2PRIM('elementwise_pow') def elementwise_pow_orig2prim(op, x, y): if x.shape != y.shape: @@ -575,14 +573,11 @@ def gt_prim2orig(op, x, y): return paddle.greater_than(x, y) -<<<<<<< 7036dbc3b81d1b4fffdf106835abdb2f40ee09d2 @REGISTER_PRIM2ORIG('ge_p') def ge_prim2orig(op, x, y): return paddle.greater_equal(x, y) -======= ->>>>>>> add ne_p gt_p primitive operators @REGISTER_PRIM2ORIG('ne_p') def ne_prim2orig(op, x, y): return paddle.not_equal(x, y) @@ -869,7 +864,7 @@ def gt_jvp(op, x_dot, y_dot): return z_dot -<<<<<<< 7036dbc3b81d1b4fffdf106835abdb2f40ee09d2 + @REGISTER_JVP('ge_p') def ge_jvp(op, x_dot, y_dot): if x_dot is None and y_dot is None: @@ -879,8 +874,6 @@ def ge_jvp(op, x_dot, y_dot): return z_dot -======= ->>>>>>> add ne_p gt_p primitive operators @REGISTER_JVP('ne_p') def ne_jvp(op, x_dot, y_dot): if x_dot is None and y_dot is None: From e56fcac3b2659970e5de2d02187c5aa8379cb35a Mon Sep 17 00:00:00 2001 From: Xiaoxu Chen Date: Wed, 14 Sep 2022 02:54:58 +0000 Subject: [PATCH 4/6] add cast primitive operators --- .../fluid/operators/prim_ops/CMakeLists.txt | 3 +- paddle/fluid/operators/prim_ops/cast_p_op.cc | 78 +++++++++++++++++++ python/paddle/incubate/autograd/primops.py | 11 +++ python/paddle/incubate/autograd/primrules.py | 40 +++++++++- 4 files changed, 130 insertions(+), 2 deletions(-) create mode 100644 paddle/fluid/operators/prim_ops/cast_p_op.cc diff --git a/paddle/fluid/operators/prim_ops/CMakeLists.txt b/paddle/fluid/operators/prim_ops/CMakeLists.txt index 9d24cf89af494..30e162a4dd2a9 100644 --- a/paddle/fluid/operators/prim_ops/CMakeLists.txt +++ b/paddle/fluid/operators/prim_ops/CMakeLists.txt @@ -36,7 +36,8 @@ set(PRIM_OP_SRCS pow_p_op.cc max_p_op.cc erf_p_op.cc - abs_p_op.cc) + abs_p_op.cc + cast_p_op.cc) cc_test( prim_op_test diff --git a/paddle/fluid/operators/prim_ops/cast_p_op.cc b/paddle/fluid/operators/prim_ops/cast_p_op.cc new file mode 100644 index 0000000000000..f61125504574c --- /dev/null +++ b/paddle/fluid/operators/prim_ops/cast_p_op.cc @@ -0,0 +1,78 @@ +// Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "paddle/fluid/framework/op_registry.h" +#include "paddle/fluid/framework/operator.h" + +namespace paddle { +namespace framework { +class InferShapeContext; +class VarDesc; +} // namespace framework +} // namespace paddle + +namespace paddle { +namespace operators { +class CastPrimOp : public framework::OperatorBase { + public: + CastPrimOp(const std::string &type, + const framework::VariableNameMap &inputs, + const framework::VariableNameMap &outputs, + const framework::AttributeMap &attrs) + : framework::OperatorBase(type, inputs, outputs, attrs) {} + void RunImpl(const framework::Scope &scope, + const platform::Place &dev_place) const override { + PADDLE_THROW(platform::errors::Unimplemented( + "Prim operator cast_p should not be excuted directly")); + } +}; + +class CastPrimOpMaker : public framework::OpProtoAndCheckerMaker { + public: + void Make() override { + AddInput("X", "(Tensor), The input tensor of cast_p op."); + AddOutput("Y", "(Tensor), The output tensor of cast_p op."); + AddAttr("dtype", "output data type"); + AddComment(R"DOC(Autograd primitive cast_p operator.)DOC"); + } +}; + +class CastPrimOpShapeInference : public framework::InferShapeBase { + public: + void operator()(framework::InferShapeContext *ctx) const override { + framework::InferShapeVarPtr x_var_ptr = ctx->GetInputVarPtrs("X")[0]; + framework::InferShapeVarPtr y_var_ptr = ctx->GetOutputVarPtrs("Y")[0]; + framework::VarDesc *x_var = PADDLE_GET(framework::VarDesc *, x_var_ptr); + PADDLE_GET(framework::VarDesc *, y_var_ptr)->SetShape(x_var->GetShape()); + } +}; + +class CastPrimOpVarTypeInference + : public framework::StaticGraphVarTypeInference { + public: + void operator()(framework::InferVarTypeContext *ctx) const override { + auto out_type = static_cast( + PADDLE_GET_CONST(int, ctx->GetAttr("dtype"))); + ctx->SetOutputDataType("Out", out_type); + } +}; + +} // namespace operators +} // namespace paddle + +REGISTER_OPERATOR(cast_p, + paddle::operators::CastPrimOp, + paddle::operators::CastPrimOpMaker, + paddle::operators::CastPrimOpShapeInference, + paddle::operators::CastPrimOpVarTypeInference); diff --git a/python/paddle/incubate/autograd/primops.py b/python/paddle/incubate/autograd/primops.py index dde3fb492cd97..d4daad8ef2d72 100644 --- a/python/paddle/incubate/autograd/primops.py +++ b/python/paddle/incubate/autograd/primops.py @@ -382,3 +382,14 @@ def max(x, y, out=None): @REGISTER_FN('erf_p', 'X', 'Y') def erf(x, out=None): return _simple_unop(LayerHelper('erf_p', **locals())) + + +@REGISTER_FN('cast_p', 'X', 'Y') +def cast(x, dtype, out=None): + helper = LayerHelper('cast_p', **locals()) + if out is None: + out = helper.create_variable_for_type_inference(dtype) + helper.append_op(type=helper.layer_type, + outputs={'Y': out}, + attrs={'dtype': dtype}) + return out diff --git a/python/paddle/incubate/autograd/primrules.py b/python/paddle/incubate/autograd/primrules.py index 6b09bb51e7a7c..42430e72d13a5 100644 --- a/python/paddle/incubate/autograd/primrules.py +++ b/python/paddle/incubate/autograd/primrules.py @@ -322,6 +322,11 @@ def num_el(shape): raise RuntimeError('Only support lower l2/l1 norm currently') +@REGISTER_ORIG2PRIM('cast') +def cast_orig2pirm(op, x): + return primops.cast(x, op.attr('out_dtype')) + + # TODO: support broadcast @REGISTER_ORIG2PRIM('where') def select_orig2prim(op, condition, x, y): @@ -357,15 +362,26 @@ def ge_orig2prim(op, x, y): return ge(x, y) +# paddle.pow API use "elementwise_pow" operator when y is a Tensor. @REGISTER_ORIG2PRIM('elementwise_pow') def elementwise_pow_orig2prim(op, x, y): if x.shape != y.shape: y = broadcast(y, shape=x.shape) - z = primops.pow(x, y) return z +# paddle.pow API use "pow" operator when y is a scalar. +@REGISTER_ORIG2PRIM('pow') +def pow_orig2prim(op, x): + return primops.pow(x, fill_const(op.attr('factor'), x.shape, x.dtype)) + + +@REGISTER_ORIG2PRIM('square') +def square_orig2prim(op, x): + return primops.pow(x, fill_const(2., x.shape, x.dtype)) + + @REGISTER_ORIG2PRIM('elementwise_max') def elementwise_max_orig2prim(op, x, y): if x.shape != y.shape: @@ -416,6 +432,11 @@ def reduce_mean_orig2prim(op, x): return div(sum, norm) +@REGISTER_ORIG2PRIM('size') +def size_orig2prim(op, x): + return fill_const(functools.reduce(operator.mul, x.shape), (1, ), x.dtype) + + ## Register prim2orig lower rules @REGISTER_PRIM2ORIG('add_p') def add_prim2orig(op, x, y): @@ -593,6 +614,11 @@ def max_prim2orig(op, x, y): return paddle.maximum(x, y) +@REGISTER_PRIM2ORIG('cast_p') +def max_prim2orig(op, x): + return paddle.cast(x, op.attr('dtype')) + + ## Register linearize rules @REGISTER_JVP('add_p') def add_jvp(op, x_dot, y_dot): @@ -930,6 +956,12 @@ def max_jvp(op, x_dot, y_dot): return select(eq(y, z), y_dot, x_dot) +@REGISTER_JVP('cast_p') +def cast_jvp(op, x_dot): + y = op_position_output(op) + return primops.cast(x_bar, y.dtype) + + ## Register transpose rules @@ -1134,3 +1166,9 @@ def select_transpose(op, check_dot, z_bar): y_bar = select(cond, zeros_y, z_bar) if check_dot(y) else None return cond_bar, x_bar, y_bar + + +@REGISTER_TRANSPOSE('cast_p') +def cast_transpose(op, check_dot, y_bar): + x, = op_position_inputs(op) + return primops.cast(y_bar, x.dtype) From b4d1c4febf9232b9e4245cc543a2b198592b6a76 Mon Sep 17 00:00:00 2001 From: Xiaoxu Chen Date: Wed, 14 Sep 2022 03:17:56 +0000 Subject: [PATCH 5/6] add pow,square prim2oirg rules --- paddle/fluid/operators/prim_ops/cast_p_op.cc | 2 +- .../autograd/test_jvp_and_transpose.py | 30 ++++++++++++++++ .../unittests/autograd/test_orig2prim.py | 36 +++++++++++++++++++ .../unittests/autograd/test_prim2orig.py | 20 +++++++++++ .../tests/unittests/autograd/test_primapi.py | 17 +++++++-- .../tests/unittests/autograd/test_primops.py | 3 ++ python/paddle/incubate/autograd/primops.py | 1 + python/paddle/incubate/autograd/primrules.py | 6 ++-- 8 files changed, 108 insertions(+), 7 deletions(-) diff --git a/paddle/fluid/operators/prim_ops/cast_p_op.cc b/paddle/fluid/operators/prim_ops/cast_p_op.cc index f61125504574c..5c8b9ab45c6bc 100644 --- a/paddle/fluid/operators/prim_ops/cast_p_op.cc +++ b/paddle/fluid/operators/prim_ops/cast_p_op.cc @@ -64,7 +64,7 @@ class CastPrimOpVarTypeInference void operator()(framework::InferVarTypeContext *ctx) const override { auto out_type = static_cast( PADDLE_GET_CONST(int, ctx->GetAttr("dtype"))); - ctx->SetOutputDataType("Out", out_type); + ctx->SetOutputDataType("Y", out_type); } }; diff --git a/python/paddle/fluid/tests/unittests/autograd/test_jvp_and_transpose.py b/python/paddle/fluid/tests/unittests/autograd/test_jvp_and_transpose.py index 6ba8cce25782b..13b800b28b48a 100644 --- a/python/paddle/fluid/tests/unittests/autograd/test_jvp_and_transpose.py +++ b/python/paddle/fluid/tests/unittests/autograd/test_jvp_and_transpose.py @@ -433,6 +433,36 @@ def init_data(self): ] +class TestCastPJVPAndTranspose(TestAddPJVPAndTranspose): + + def init_data(self): + # Set prim op + self.op_type = 'cast_p' + X = paddle.static.data(name='X', shape=[5, 6], dtype='int64') + self.prim_input = { + 'X': X, + } + self.prim_output = { + 'Y': + self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + } + self.prim_attrs = {'dtype': paddle.float64} + + # Set JVP + X_DOT = paddle.static.data(name='X_DOT', shape=[5, 6], dtype='int64') + self.jvp_args = (X_DOT, ) + self.jvp_out_shape_map = {0: self.prim_output['Y']} + + self.all_ops = [ + # prim op: + 'cast_p', + # jvp op: + 'cast_p', + # transpose op: + 'cast_p' + ] + + class TestLogPJVPAndTranspose(TestAddPJVPAndTranspose): def init_data(self): diff --git a/python/paddle/fluid/tests/unittests/autograd/test_orig2prim.py b/python/paddle/fluid/tests/unittests/autograd/test_orig2prim.py index 92a50d8bb1b08..573f38caaad98 100644 --- a/python/paddle/fluid/tests/unittests/autograd/test_orig2prim.py +++ b/python/paddle/fluid/tests/unittests/autograd/test_orig2prim.py @@ -786,5 +786,41 @@ def init_data(self): self.out_map = {0: self.output['Out']} +class TestSizeOrig2Prim(TestElementWiseAddOrig2Prim): + + def init_data(self): + self.op_type = 'size' + X = paddle.static.data(name='X', shape=[5, 8], dtype='float') + + self.input = {'X': X} + self.output = { + 'Out': + self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + } + + self.orig2prim_args = (X, ) + self.all_ops = ['size', 'fill_constant_p'] + # { prim_op_output_index: orig_op_output_var } + self.out_map = {0: self.output['Out']} + + +class TestCastOrig2Prim(TestElementWiseAddOrig2Prim): + + def init_data(self): + self.op_type = 'cast' + X = paddle.static.data(name='X', shape=[5, 8], dtype='float') + + self.input = {'X': X} + self.output = { + 'Out': + self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + } + self.attrs = {'dtype': paddle.float64} + self.orig2prim_args = (X, ) + self.all_ops = ['cast', 'cast_p'] + # { prim_op_output_index: orig_op_output_var } + self.out_map = {0: self.output['Out']} + + if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/autograd/test_prim2orig.py b/python/paddle/fluid/tests/unittests/autograd/test_prim2orig.py index c173cc4790dc2..a89b91bdd2b64 100644 --- a/python/paddle/fluid/tests/unittests/autograd/test_prim2orig.py +++ b/python/paddle/fluid/tests/unittests/autograd/test_prim2orig.py @@ -670,5 +670,25 @@ def init_data(self): self.out_map = {self.output['Z']: 0} +class TestCastPPrim2Orig(TestAddPPrim2Orig): + + def init_data(self): + self.op_type = 'cast_p' + X = paddle.static.data(name='X', shape=[7, 8], dtype='float64') + + self.input = { + 'X': X, + } + self.output = { + 'Y': + self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + } + self.attrs = {'dtype': paddle.int64} + + self.prim2orig_args = (X, ) + self.all_ops = ['cast_p', 'cast'] + self.out_map = {self.output['Y']: 0} + + if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/autograd/test_primapi.py b/python/paddle/fluid/tests/unittests/autograd/test_primapi.py index f518685108ab6..ebca658c0f69d 100644 --- a/python/paddle/fluid/tests/unittests/autograd/test_primapi.py +++ b/python/paddle/fluid/tests/unittests/autograd/test_primapi.py @@ -300,8 +300,21 @@ def test_illegal_param(self): (np.random.rand(200, 345), ), None, 'float32'), ('abs', paddle.abs, (np.random.uniform(-10, 10, (200, 345)), ), None, 'float32'), - )) - + ('cast_float', lambda x: paddle.cast(x, paddle.float64), + (np.random.rand(10, 20), ), None, 'float32'), + ('cast_int', lambda x: paddle.cast(x, paddle.int32), + (np.random.rand(10, 20), ), None, 'float32'), + ('square', paddle.square, (np.random.rand(100), ), None, 'float32'), + ('pow_scalar', lambda x: paddle.pow(x, 2), + (np.random.rand(20, 30), ), None, 'float32'), + ('var', paddle.var, (np.random.rand(200, 324), ), None, 'float32'), + ('var_with_axis', lambda x: paddle.var(x, axis=1), + (np.random.rand(10, 20, 30), ), None, 'float32'), + ('var_without_unbiased', + lambda x: paddle.var(x, axis=1, unbiased=False), + (np.random.rand(10, 20, 30), ), None, 'float32'), + ('var_with_keepdim', lambda x: paddle.var(x, axis=1, keepdim=True), + (np.random.rand(10, 20, 30), ), None, 'float32'))) class TestGrad(unittest.TestCase): def setUp(self): diff --git a/python/paddle/fluid/tests/unittests/autograd/test_primops.py b/python/paddle/fluid/tests/unittests/autograd/test_primops.py index ba6f094e68008..a8668920e6fbe 100644 --- a/python/paddle/fluid/tests/unittests/autograd/test_primops.py +++ b/python/paddle/fluid/tests/unittests/autograd/test_primops.py @@ -44,6 +44,9 @@ ('erf', primops.erf, randn(2, 3), {}, (2, 3), 'float64'), ('abs', primops.abs, randn(2, 3), {}, (2, 3), 'float64'), ('log', primops.log, randn(2, 3), {}, (2, 3), 'float64'), + ('cast', primops.cast, randn(2, 3), { + 'dtype': paddle.int64 + }, 'int64'), ('reshape', primops.reshape, randn(2, 3), { 'shape': (3, 2) }, (3, 2), 'float64'), diff --git a/python/paddle/incubate/autograd/primops.py b/python/paddle/incubate/autograd/primops.py index d4daad8ef2d72..636dc89220490 100644 --- a/python/paddle/incubate/autograd/primops.py +++ b/python/paddle/incubate/autograd/primops.py @@ -390,6 +390,7 @@ def cast(x, dtype, out=None): if out is None: out = helper.create_variable_for_type_inference(dtype) helper.append_op(type=helper.layer_type, + inputs={'X': x}, outputs={'Y': out}, attrs={'dtype': dtype}) return out diff --git a/python/paddle/incubate/autograd/primrules.py b/python/paddle/incubate/autograd/primrules.py index 42430e72d13a5..e28856839caa8 100644 --- a/python/paddle/incubate/autograd/primrules.py +++ b/python/paddle/incubate/autograd/primrules.py @@ -323,8 +323,8 @@ def num_el(shape): @REGISTER_ORIG2PRIM('cast') -def cast_orig2pirm(op, x): - return primops.cast(x, op.attr('out_dtype')) +def cast_orig2prim(op, x): + return primops.cast(x, paddle.dtype(op.attr('out_dtype'))) # TODO: support broadcast @@ -354,7 +354,6 @@ def gt_orig2prim(op, x, y): return gt(x, y) - @REGISTER_ORIG2PRIM('greater_equal') def ge_orig2prim(op, x, y): if x.shape != y.shape: @@ -890,7 +889,6 @@ def gt_jvp(op, x_dot, y_dot): return z_dot - @REGISTER_JVP('ge_p') def ge_jvp(op, x_dot, y_dot): if x_dot is None and y_dot is None: From 3c41f0a4bdc90ae5081dc3a9acfb9d5a1eac110d Mon Sep 17 00:00:00 2001 From: Xiaoxu Chen Date: Wed, 14 Sep 2022 06:43:15 +0000 Subject: [PATCH 6/6] add elementwise_div orig2prim rule --- .../autograd/test_jvp_and_transpose.py | 34 ++-------- .../unittests/autograd/test_orig2prim.py | 65 +++++++++++++++++-- .../tests/unittests/autograd/test_primapi.py | 2 + .../tests/unittests/autograd/test_primops.py | 2 +- python/paddle/incubate/autograd/primreg.py | 2 +- python/paddle/incubate/autograd/primrules.py | 21 ++++-- 6 files changed, 86 insertions(+), 40 deletions(-) diff --git a/python/paddle/fluid/tests/unittests/autograd/test_jvp_and_transpose.py b/python/paddle/fluid/tests/unittests/autograd/test_jvp_and_transpose.py index 13b800b28b48a..c8e1b3965228d 100644 --- a/python/paddle/fluid/tests/unittests/autograd/test_jvp_and_transpose.py +++ b/python/paddle/fluid/tests/unittests/autograd/test_jvp_and_transpose.py @@ -453,6 +453,12 @@ def init_data(self): self.jvp_args = (X_DOT, ) self.jvp_out_shape_map = {0: self.prim_output['Y']} + # Set transpose + check_dot = lambda v: True + Y_BAR = paddle.static.data(name='Y_BAR', shape=[5, 6], dtype='float') + self.transpose_args = (check_dot, Y_BAR) + self.transpose_out_shape_map = {0: X} + self.all_ops = [ # prim op: 'cast_p', @@ -1070,34 +1076,6 @@ def init_data(self): # transpose op: ] -class TestGePJVPAndTranspose(TestAddPJVPAndTranspose): - - def init_data(self): - # Set prim op - self.op_type = 'ge_p' - X = paddle.static.data(name='X', shape=[4, 5], dtype='float64') - Y = paddle.static.data(name='Y', shape=[4, 5], dtype='float64') - - self.prim_input = {'X': X, 'Y': Y} - self.prim_output = { - 'Z': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) - } - self.prim_attrs = {} - - # Set JVP - X_DOT = paddle.static.data(name='X_DOT', shape=[4, 5], dtype='float64') - Y_DOT = paddle.static.data(name='Y_DOT', shape=[4, 5], dtype='float64') - self.jvp_args = (X_DOT, Y_DOT) - self.jvp_out_shape_map = {0: self.prim_output['Z']} - - self.all_ops = [ - # prim op: - 'ge_p', - # jvp op: - 'fill_constant_p', - # transpose op: - ] class TestGePJVPAndTranspose(TestAddPJVPAndTranspose): diff --git a/python/paddle/fluid/tests/unittests/autograd/test_orig2prim.py b/python/paddle/fluid/tests/unittests/autograd/test_orig2prim.py index 573f38caaad98..e1d5ee11a13ac 100644 --- a/python/paddle/fluid/tests/unittests/autograd/test_orig2prim.py +++ b/python/paddle/fluid/tests/unittests/autograd/test_orig2prim.py @@ -110,6 +110,26 @@ def init_data(self): self.out_map = {0: self.output['Out']} +class TestElementWiseDivOrig2Prim(TestElementWiseAddOrig2Prim): + + def init_data(self): + self.op_type = 'elementwise_div' + X = paddle.static.data(name='X', shape=[8, 8], dtype='float') + Y = paddle.static.data(name='Y', shape=[8, 8], dtype='float') + + self.input = {'X': X, 'Y': Y} + self.output = { + 'Out': + self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + } + self.attrs = {} + + self.orig2prim_args = (X, Y) + self.all_ops = ['elementwise_div', 'div_p'] + # { prim_op_output_index: orig_op_output_var } + self.out_map = {0: self.output['Out']} + + class TestMatmulV2Orig2Prim(TestElementWiseAddOrig2Prim): def init_data(self): @@ -792,12 +812,13 @@ def init_data(self): self.op_type = 'size' X = paddle.static.data(name='X', shape=[5, 8], dtype='float') - self.input = {'X': X} + self.input = {'Input': X} self.output = { 'Out': - self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + self.layer_help.create_variable_for_type_inference( + dtype=paddle.int64) } - + self.attrs = {} self.orig2prim_args = (X, ) self.all_ops = ['size', 'fill_constant_p'] # { prim_op_output_index: orig_op_output_var } @@ -815,12 +836,48 @@ def init_data(self): 'Out': self.layer_help.create_variable_for_type_inference(dtype=X.dtype) } - self.attrs = {'dtype': paddle.float64} + self.attrs = {'in_dtype': X.dtype, 'out_dtype': paddle.float64} self.orig2prim_args = (X, ) self.all_ops = ['cast', 'cast_p'] # { prim_op_output_index: orig_op_output_var } self.out_map = {0: self.output['Out']} +class TestPowScalarOrig2Prim(TestElementWiseAddOrig2Prim): + + def init_data(self): + self.op_type = 'pow' + X = paddle.static.data(name='X', shape=[5, 8], dtype='float') + + self.input = {'X': X} + self.output = { + 'Out': + self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + } + self.attrs = {'factor': 2.} + self.orig2prim_args = (None, X) + self.all_ops = ['pow', 'pow_p', 'fill_constant_p'] + # { prim_op_output_index: orig_op_output_var } + self.out_map = {0: self.output['Out']} + + +class TestSquareOrig2Prim(TestElementWiseAddOrig2Prim): + + def init_data(self): + self.op_type = 'square' + X = paddle.static.data(name='X', shape=[5, 8], dtype='float') + + self.input = {'X': X} + self.output = { + 'Out': + self.layer_help.create_variable_for_type_inference(dtype=X.dtype) + } + self.attrs = {} + self.orig2prim_args = (X, ) + self.all_ops = ['square', 'pow_p', 'fill_constant_p'] + # { prim_op_output_index: orig_op_output_var } + self.out_map = {0: self.output['Out']} + + if __name__ == '__main__': unittest.main() diff --git a/python/paddle/fluid/tests/unittests/autograd/test_primapi.py b/python/paddle/fluid/tests/unittests/autograd/test_primapi.py index ebca658c0f69d..bdc54563fc8d2 100644 --- a/python/paddle/fluid/tests/unittests/autograd/test_primapi.py +++ b/python/paddle/fluid/tests/unittests/autograd/test_primapi.py @@ -257,6 +257,8 @@ def test_illegal_param(self): (np.random.rand(2, 3), np.random.rand(3, 2)), None, 'float32'), ('multiply', paddle.multiply, (np.random.rand(2, 3), np.random.rand(2, 3)), None, 'float64'), + ('div', paddle.divide, + (np.random.rand(2, 3), np.random.rand(2, 3)), None, 'float64'), ('add', paddle.add, (np.random.rand(2, 3), np.random.rand(2, 3)), None, 'float32'), ('input_not_sequence', paddle.tanh, diff --git a/python/paddle/fluid/tests/unittests/autograd/test_primops.py b/python/paddle/fluid/tests/unittests/autograd/test_primops.py index a8668920e6fbe..35291432f6e8f 100644 --- a/python/paddle/fluid/tests/unittests/autograd/test_primops.py +++ b/python/paddle/fluid/tests/unittests/autograd/test_primops.py @@ -46,7 +46,7 @@ ('log', primops.log, randn(2, 3), {}, (2, 3), 'float64'), ('cast', primops.cast, randn(2, 3), { 'dtype': paddle.int64 - }, 'int64'), + }, (2, 3), 'int64'), ('reshape', primops.reshape, randn(2, 3), { 'shape': (3, 2) }, (3, 2), 'float64'), diff --git a/python/paddle/incubate/autograd/primreg.py b/python/paddle/incubate/autograd/primreg.py index 4721500b2be09..7972409d9356c 100644 --- a/python/paddle/incubate/autograd/primreg.py +++ b/python/paddle/incubate/autograd/primreg.py @@ -80,7 +80,7 @@ def div(x, y, out=None): """ args = _primop_position_argnames.lookup(op.type) - assert args is not None, 'args should not be None in op_position_inputs().' + assert args is not None, f'args of {op.type} should not be None in op_position_inputs().' *input_names, _ = args inputs = [] diff --git a/python/paddle/incubate/autograd/primrules.py b/python/paddle/incubate/autograd/primrules.py index e28856839caa8..4625cfd362f07 100644 --- a/python/paddle/incubate/autograd/primrules.py +++ b/python/paddle/incubate/autograd/primrules.py @@ -158,6 +158,13 @@ def elementwise_mul_orig2prim(op, x, y): return z +@REGISTER_ORIG2PRIM('elementwise_div') +def elementwise_div_orig2prim(op, x, y): + if x.shape != y.shape: + y = broadcast(y, shape=x.shape) + return primops.div(x, y) + + @REGISTER_ORIG2PRIM('tanh') def tanh_orig2prim(op, x): return tanh(x) @@ -372,8 +379,9 @@ def elementwise_pow_orig2prim(op, x, y): # paddle.pow API use "pow" operator when y is a scalar. @REGISTER_ORIG2PRIM('pow') -def pow_orig2prim(op, x): - return primops.pow(x, fill_const(op.attr('factor'), x.shape, x.dtype)) +def pow_orig2prim(op, x, y): + # x is factorTensor defined in paddle phi op. Currently it is None. + return primops.pow(y, fill_const(op.attr('factor'), y.shape, y.dtype)) @REGISTER_ORIG2PRIM('square') @@ -433,7 +441,8 @@ def reduce_mean_orig2prim(op, x): @REGISTER_ORIG2PRIM('size') def size_orig2prim(op, x): - return fill_const(functools.reduce(operator.mul, x.shape), (1, ), x.dtype) + return fill_const(functools.reduce(operator.mul, x.shape), (1, ), + paddle.int64) ## Register prim2orig lower rules @@ -614,8 +623,8 @@ def max_prim2orig(op, x, y): @REGISTER_PRIM2ORIG('cast_p') -def max_prim2orig(op, x): - return paddle.cast(x, op.attr('dtype')) +def cast_prim2orig(op, x): + return paddle.cast(x, paddle.dtype(op.attr('dtype'))) ## Register linearize rules @@ -957,7 +966,7 @@ def max_jvp(op, x_dot, y_dot): @REGISTER_JVP('cast_p') def cast_jvp(op, x_dot): y = op_position_output(op) - return primops.cast(x_bar, y.dtype) + return primops.cast(x_dot, y.dtype) ## Register transpose rules