From 5e60a8039a69df3299cfb274f2b137d80625c475 Mon Sep 17 00:00:00 2001 From: Netpunk <2327994230@qq.com> Date: Mon, 27 Feb 2023 17:20:24 +0800 Subject: [PATCH 01/11] support fp16 for maxout op --- paddle/phi/kernels/funcs/maxouting.cc | 2 + paddle/phi/kernels/funcs/maxouting.cu | 2 + paddle/phi/kernels/gpu/maxout_grad_kernel.cu | 2 +- paddle/phi/kernels/gpu/maxout_kernel.cu | 2 +- .../fluid/tests/unittests/test_maxout_op.py | 45 +++++++++++++++++++ python/paddle/nn/functional/activation.py | 2 +- 6 files changed, 52 insertions(+), 3 deletions(-) diff --git a/paddle/phi/kernels/funcs/maxouting.cc b/paddle/phi/kernels/funcs/maxouting.cc index 67a924128c167..e9b61e0320776 100644 --- a/paddle/phi/kernels/funcs/maxouting.cc +++ b/paddle/phi/kernels/funcs/maxouting.cc @@ -108,8 +108,10 @@ void MaxOutGradFunctor::operator()( } template class MaxOutGradFunctor; +template class MaxOutGradFunctor; template class MaxOutGradFunctor; template class MaxOutFunctor; +template class MaxOutFunctor; template class MaxOutFunctor; } // namespace funcs diff --git a/paddle/phi/kernels/funcs/maxouting.cu b/paddle/phi/kernels/funcs/maxouting.cu index 89450dbd5c60b..146bb1aca4c1b 100644 --- a/paddle/phi/kernels/funcs/maxouting.cu +++ b/paddle/phi/kernels/funcs/maxouting.cu @@ -175,9 +175,11 @@ void MaxOutGradFunctor::operator()( } template class MaxOutGradFunctor; +template class MaxOutGradFunctor; template class MaxOutGradFunctor; template class MaxOutFunctor; +template class MaxOutFunctor; template class MaxOutFunctor; } // namespace funcs diff --git a/paddle/phi/kernels/gpu/maxout_grad_kernel.cu b/paddle/phi/kernels/gpu/maxout_grad_kernel.cu index a405f38523a75..283eda9b817b5 100644 --- a/paddle/phi/kernels/gpu/maxout_grad_kernel.cu +++ b/paddle/phi/kernels/gpu/maxout_grad_kernel.cu @@ -16,4 +16,4 @@ #include "paddle/phi/kernels/impl/maxout_grad_kernel_impl.h" PD_REGISTER_KERNEL( - maxout_grad, GPU, ALL_LAYOUT, phi::MaxOutGradKernel, float, double) {} + maxout_grad, GPU, ALL_LAYOUT, phi::MaxOutGradKernel, float, phi::dtype::float16, double) {} diff --git a/paddle/phi/kernels/gpu/maxout_kernel.cu b/paddle/phi/kernels/gpu/maxout_kernel.cu index e5407a4925c84..e86cdef8e2a26 100644 --- a/paddle/phi/kernels/gpu/maxout_kernel.cu +++ b/paddle/phi/kernels/gpu/maxout_kernel.cu @@ -15,4 +15,4 @@ #include "paddle/phi/core/kernel_registry.h" #include "paddle/phi/kernels/impl/maxout_kernel_impl.h" -PD_REGISTER_KERNEL(maxout, GPU, ALL_LAYOUT, phi::MaxOutKernel, float, double) {} +PD_REGISTER_KERNEL(maxout, GPU, ALL_LAYOUT, phi::MaxOutKernel, float, phi::dtype::float16, double) {} diff --git a/python/paddle/fluid/tests/unittests/test_maxout_op.py b/python/paddle/fluid/tests/unittests/test_maxout_op.py index 7756f7d4ae841..5ea604f7c8f8c 100644 --- a/python/paddle/fluid/tests/unittests/test_maxout_op.py +++ b/python/paddle/fluid/tests/unittests/test_maxout_op.py @@ -136,5 +136,50 @@ def test_errors(self): self.assertRaises(ValueError, F.maxout, x_float32, 2, 2) +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) +class TestMaxOutOpFP16(OpTest): + def setUp(self): + self.op_type = "maxout" + self.python_api = paddle.nn.Maxout + input_np = np.random.uniform(-1, 1, [2, 6, 5, 4]).astype(np.float16) + self.groups = 2 + self.axis = 1 + output_np = maxout_forward_naive(input_np, self.groups, self.axis) + self.attrs = { + 'groups': self.groups, + 'axis': self.axis + } + self.inputs = {'X': input_np} + self.outputs = {'Out': output_np} + + def test_check_output(self): + if core.is_compiled_with_cuda(): + place = core.CUDAPlace(0) + if core.is_float16_supported(place): + self.check_output_with_place(place, atol=1e-3) + + def test_check_grad(self): + place = core.CUDAPlace(0) + if core.is_float16_supported(place): + self.check_grad_with_place( + place, ['X'], 'Out', max_relative_error=0.5 + ) + + def set_attrs(self): + pass + + +class TestMaxoutFP16Case1(TestMaxOutOpFP16): + def set_attrs(self): + self.axis = -1 + + +class TestMaxoutFP16Case2(TestMaxOutOpFP16): + def set_attrs(self): + self.axis = 3 + + if __name__ == '__main__': unittest.main() diff --git a/python/paddle/nn/functional/activation.py b/python/paddle/nn/functional/activation.py index 569a533448600..d105df1182f2f 100644 --- a/python/paddle/nn/functional/activation.py +++ b/python/paddle/nn/functional/activation.py @@ -784,7 +784,7 @@ def maxout(x, groups, axis=1, name=None): Parameters: x (Tensor): The input is 4-D Tensor with shape [N, C, H, W] or [N, H, W, C], the data type - of input is float32 or float64. + of input is float16, float32 or float64. groups (int): The groups number of maxout. `groups` specifies the index of channel dimension where maxout will be performed. This must be a factor of number of features. From fa549ac3376dbe387dee1f39ca9e647905d66ee6 Mon Sep 17 00:00:00 2001 From: Netpunk <2327994230@qq.com> Date: Tue, 28 Feb 2023 18:01:23 +0800 Subject: [PATCH 02/11] format code --- paddle/phi/kernels/gpu/maxout_grad_kernel.cu | 9 +++++++-- paddle/phi/kernels/gpu/maxout_kernel.cu | 8 +++++++- python/paddle/fluid/tests/unittests/test_maxout_op.py | 5 +---- 3 files changed, 15 insertions(+), 7 deletions(-) diff --git a/paddle/phi/kernels/gpu/maxout_grad_kernel.cu b/paddle/phi/kernels/gpu/maxout_grad_kernel.cu index 283eda9b817b5..7d59436019c71 100644 --- a/paddle/phi/kernels/gpu/maxout_grad_kernel.cu +++ b/paddle/phi/kernels/gpu/maxout_grad_kernel.cu @@ -15,5 +15,10 @@ #include "paddle/phi/core/kernel_registry.h" #include "paddle/phi/kernels/impl/maxout_grad_kernel_impl.h" -PD_REGISTER_KERNEL( - maxout_grad, GPU, ALL_LAYOUT, phi::MaxOutGradKernel, float, phi::dtype::float16, double) {} +PD_REGISTER_KERNEL(maxout_grad, + GPU, + ALL_LAYOUT, + phi::MaxOutGradKernel, + float, + phi::dtype::float16, + double) {} diff --git a/paddle/phi/kernels/gpu/maxout_kernel.cu b/paddle/phi/kernels/gpu/maxout_kernel.cu index e86cdef8e2a26..4871046450264 100644 --- a/paddle/phi/kernels/gpu/maxout_kernel.cu +++ b/paddle/phi/kernels/gpu/maxout_kernel.cu @@ -15,4 +15,10 @@ #include "paddle/phi/core/kernel_registry.h" #include "paddle/phi/kernels/impl/maxout_kernel_impl.h" -PD_REGISTER_KERNEL(maxout, GPU, ALL_LAYOUT, phi::MaxOutKernel, float, phi::dtype::float16, double) {} +PD_REGISTER_KERNEL(maxout, + GPU, + ALL_LAYOUT, + phi::MaxOutKernel, + float, + phi::dtype::float16, + double) {} diff --git a/python/paddle/fluid/tests/unittests/test_maxout_op.py b/python/paddle/fluid/tests/unittests/test_maxout_op.py index 5ea604f7c8f8c..8900b0ae5fd53 100644 --- a/python/paddle/fluid/tests/unittests/test_maxout_op.py +++ b/python/paddle/fluid/tests/unittests/test_maxout_op.py @@ -147,10 +147,7 @@ def setUp(self): self.groups = 2 self.axis = 1 output_np = maxout_forward_naive(input_np, self.groups, self.axis) - self.attrs = { - 'groups': self.groups, - 'axis': self.axis - } + self.attrs = {'groups': self.groups, 'axis': self.axis} self.inputs = {'X': input_np} self.outputs = {'Out': output_np} From cd8f23d79cdeba25b3c4515d3ee6a0ef49d8a9b3 Mon Sep 17 00:00:00 2001 From: Netpunk <2327994230@qq.com> Date: Mon, 6 Mar 2023 13:56:41 +0800 Subject: [PATCH 03/11] change api --- python/paddle/fluid/tests/unittests/test_maxout_op.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/fluid/tests/unittests/test_maxout_op.py b/python/paddle/fluid/tests/unittests/test_maxout_op.py index 8900b0ae5fd53..1d6bb082b5f96 100644 --- a/python/paddle/fluid/tests/unittests/test_maxout_op.py +++ b/python/paddle/fluid/tests/unittests/test_maxout_op.py @@ -142,7 +142,7 @@ def test_errors(self): class TestMaxOutOpFP16(OpTest): def setUp(self): self.op_type = "maxout" - self.python_api = paddle.nn.Maxout + self.python_api = paddle.nn.functional.maxout input_np = np.random.uniform(-1, 1, [2, 6, 5, 4]).astype(np.float16) self.groups = 2 self.axis = 1 From bf5d25007fe6fac56fac0f43d80eede3c7c775b7 Mon Sep 17 00:00:00 2001 From: Netpunk <2327994230@qq.com> Date: Thu, 9 Mar 2023 17:25:48 +0800 Subject: [PATCH 04/11] add test for static float16 --- paddle/phi/kernels/funcs/maxouting.cc | 2 -- .../fluid/tests/unittests/test_maxout_op.py | 35 ++++++++++++++----- python/paddle/nn/functional/activation.py | 2 +- 3 files changed, 28 insertions(+), 11 deletions(-) diff --git a/paddle/phi/kernels/funcs/maxouting.cc b/paddle/phi/kernels/funcs/maxouting.cc index e9b61e0320776..67a924128c167 100644 --- a/paddle/phi/kernels/funcs/maxouting.cc +++ b/paddle/phi/kernels/funcs/maxouting.cc @@ -108,10 +108,8 @@ void MaxOutGradFunctor::operator()( } template class MaxOutGradFunctor; -template class MaxOutGradFunctor; template class MaxOutGradFunctor; template class MaxOutFunctor; -template class MaxOutFunctor; template class MaxOutFunctor; } // namespace funcs diff --git a/python/paddle/fluid/tests/unittests/test_maxout_op.py b/python/paddle/fluid/tests/unittests/test_maxout_op.py index 1d6bb082b5f96..dd957d87361ea 100644 --- a/python/paddle/fluid/tests/unittests/test_maxout_op.py +++ b/python/paddle/fluid/tests/unittests/test_maxout_op.py @@ -152,17 +152,15 @@ def setUp(self): self.outputs = {'Out': output_np} def test_check_output(self): - if core.is_compiled_with_cuda(): - place = core.CUDAPlace(0) - if core.is_float16_supported(place): - self.check_output_with_place(place, atol=1e-3) + place = core.CUDAPlace(0) + self.check_output_with_place(place) def test_check_grad(self): place = core.CUDAPlace(0) - if core.is_float16_supported(place): - self.check_grad_with_place( - place, ['X'], 'Out', max_relative_error=0.5 - ) + print(1) + self.check_grad_with_place( + place, ['X'], 'Out', max_relative_error=0.001 + ) def set_attrs(self): pass @@ -178,5 +176,26 @@ def set_attrs(self): self.axis = 3 +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) +class TestMaxoutStaticAPIFP16(unittest.TestCase): + # test paddle.nn.Maxout, paddle.nn.functional.maxout + def setUp(self): + self.x_np = np.random.uniform(-1, 1, [2, 6, 5, 4]).astype(np.float16) + self.groups = 2 + self.axis = 1 + self.place = paddle.CUDAPlace(0) + + def test_static_api(self): + with paddle.static.program_guard(paddle.static.Program()): + x = paddle.fluid.data('X', self.x_np.shape, self.x_np.dtype) + out = F.maxout(x, self.groups, self.axis) + exe = paddle.static.Executor(self.place) + res = exe.run(feed={'X': self.x_np}, fetch_list=[out]) + out_ref = maxout_forward_naive(self.x_np, self.groups, self.axis) + np.testing.assert_allclose(out_ref, res[0], rtol=1e-05) + + if __name__ == '__main__': unittest.main() diff --git a/python/paddle/nn/functional/activation.py b/python/paddle/nn/functional/activation.py index d105df1182f2f..c4d70bfd76ac6 100644 --- a/python/paddle/nn/functional/activation.py +++ b/python/paddle/nn/functional/activation.py @@ -819,7 +819,7 @@ def maxout(x, groups, axis=1, name=None): if in_dygraph_mode(): return _C_ops.maxout(x, groups, axis) else: - check_variable_and_dtype(x, 'x', ['float32', 'float64'], 'maxout') + check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], 'maxout') if axis not in [1, -1, 3]: raise ValueError( "Attr(axis) should be 1 when data format is NCHW, -1 or 3 when data format is NHWC. Received " From 9befc8dd239ee64dc17a106dbbec8617efb2859a Mon Sep 17 00:00:00 2001 From: Netpunk <2327994230@qq.com> Date: Fri, 10 Mar 2023 19:09:52 +0800 Subject: [PATCH 05/11] format code --- python/paddle/nn/functional/activation.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/python/paddle/nn/functional/activation.py b/python/paddle/nn/functional/activation.py index c4d70bfd76ac6..6c4114251900c 100644 --- a/python/paddle/nn/functional/activation.py +++ b/python/paddle/nn/functional/activation.py @@ -819,7 +819,9 @@ def maxout(x, groups, axis=1, name=None): if in_dygraph_mode(): return _C_ops.maxout(x, groups, axis) else: - check_variable_and_dtype(x, 'x', ['float16', 'float32', 'float64'], 'maxout') + check_variable_and_dtype( + x, 'x', ['float16', 'float32', 'float64'], 'maxout' + ) if axis not in [1, -1, 3]: raise ValueError( "Attr(axis) should be 1 when data format is NCHW, -1 or 3 when data format is NHWC. Received " From c35527e173a38b95dcf324ac169b372b95eb83b5 Mon Sep 17 00:00:00 2001 From: Netpunk <2327994230@qq.com> Date: Mon, 13 Mar 2023 13:19:28 +0800 Subject: [PATCH 06/11] formatting code --- .../fluid/tests/unittests/test_maxout_op.py | 21 ++++--------------- 1 file changed, 4 insertions(+), 17 deletions(-) diff --git a/python/paddle/fluid/tests/unittests/test_maxout_op.py b/python/paddle/fluid/tests/unittests/test_maxout_op.py index dd957d87361ea..e8de7bc2589e5 100644 --- a/python/paddle/fluid/tests/unittests/test_maxout_op.py +++ b/python/paddle/fluid/tests/unittests/test_maxout_op.py @@ -139,17 +139,9 @@ def test_errors(self): @unittest.skipIf( not core.is_compiled_with_cuda(), "core is not compiled with CUDA" ) -class TestMaxOutOpFP16(OpTest): - def setUp(self): - self.op_type = "maxout" - self.python_api = paddle.nn.functional.maxout - input_np = np.random.uniform(-1, 1, [2, 6, 5, 4]).astype(np.float16) - self.groups = 2 - self.axis = 1 - output_np = maxout_forward_naive(input_np, self.groups, self.axis) - self.attrs = {'groups': self.groups, 'axis': self.axis} - self.inputs = {'X': input_np} - self.outputs = {'Out': output_np} +class TestMaxOutOpFP16(TestMaxOutOp): + def set_attrs(self): + self.dtype = 'float16' def test_check_output(self): place = core.CUDAPlace(0) @@ -157,14 +149,10 @@ def test_check_output(self): def test_check_grad(self): place = core.CUDAPlace(0) - print(1) self.check_grad_with_place( place, ['X'], 'Out', max_relative_error=0.001 ) - def set_attrs(self): - pass - class TestMaxoutFP16Case1(TestMaxOutOpFP16): def set_attrs(self): @@ -180,7 +168,6 @@ def set_attrs(self): not core.is_compiled_with_cuda(), "core is not compiled with CUDA" ) class TestMaxoutStaticAPIFP16(unittest.TestCase): - # test paddle.nn.Maxout, paddle.nn.functional.maxout def setUp(self): self.x_np = np.random.uniform(-1, 1, [2, 6, 5, 4]).astype(np.float16) self.groups = 2 @@ -189,7 +176,7 @@ def setUp(self): def test_static_api(self): with paddle.static.program_guard(paddle.static.Program()): - x = paddle.fluid.data('X', self.x_np.shape, self.x_np.dtype) + x = paddle.static.data('X', self.x_np.shape, self.x_np.dtype) out = F.maxout(x, self.groups, self.axis) exe = paddle.static.Executor(self.place) res = exe.run(feed={'X': self.x_np}, fetch_list=[out]) From 32ffdc2b548a0e92c252d1243073fc5ba425ff07 Mon Sep 17 00:00:00 2001 From: Netpunk <2327994230@qq.com> Date: Tue, 18 Apr 2023 14:35:48 +0800 Subject: [PATCH 07/11] atol alignment --- python/paddle/fluid/tests/unittests/test_maxout_op.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/paddle/fluid/tests/unittests/test_maxout_op.py b/python/paddle/fluid/tests/unittests/test_maxout_op.py index e8de7bc2589e5..f65e1f0e04901 100644 --- a/python/paddle/fluid/tests/unittests/test_maxout_op.py +++ b/python/paddle/fluid/tests/unittests/test_maxout_op.py @@ -150,8 +150,7 @@ def test_check_output(self): def test_check_grad(self): place = core.CUDAPlace(0) self.check_grad_with_place( - place, ['X'], 'Out', max_relative_error=0.001 - ) + place, ['X'], 'Out') class TestMaxoutFP16Case1(TestMaxOutOpFP16): From 9356ea637540744fcee6345a7259dbdec307e3b5 Mon Sep 17 00:00:00 2001 From: Netpunk <2327994230@qq.com> Date: Wed, 19 Apr 2023 18:50:26 +0800 Subject: [PATCH 08/11] =?UTF-8?q?experiment=E2=80=941?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- python/paddle/fluid/tests/unittests/test_maxout_op.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/python/paddle/fluid/tests/unittests/test_maxout_op.py b/python/paddle/fluid/tests/unittests/test_maxout_op.py index 1af7b77fbe75c..4fca7cb818f4b 100644 --- a/python/paddle/fluid/tests/unittests/test_maxout_op.py +++ b/python/paddle/fluid/tests/unittests/test_maxout_op.py @@ -143,15 +143,6 @@ class TestMaxOutOpFP16(TestMaxOutOp): def set_attrs(self): self.dtype = 'float16' - def test_check_output(self): - place = core.CUDAPlace(0) - self.check_output_with_place(place) - - def test_check_grad(self): - place = core.CUDAPlace(0) - self.check_grad_with_place( - place, ['X'], 'Out') - class TestMaxoutFP16Case1(TestMaxOutOpFP16): def set_attrs(self): From 16fe004d0999467dae354546797173b15fd43a9c Mon Sep 17 00:00:00 2001 From: Netpunk <2327994230@qq.com> Date: Wed, 19 Apr 2023 22:57:18 +0800 Subject: [PATCH 09/11] experiment-2 --- python/paddle/fluid/tests/unittests/test_maxout_op.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/python/paddle/fluid/tests/unittests/test_maxout_op.py b/python/paddle/fluid/tests/unittests/test_maxout_op.py index 4fca7cb818f4b..66fabe4929180 100644 --- a/python/paddle/fluid/tests/unittests/test_maxout_op.py +++ b/python/paddle/fluid/tests/unittests/test_maxout_op.py @@ -154,9 +154,6 @@ def set_attrs(self): self.axis = 3 -@unittest.skipIf( - not core.is_compiled_with_cuda(), "core is not compiled with CUDA" -) class TestMaxoutStaticAPIFP16(unittest.TestCase): def setUp(self): self.x_np = np.random.uniform(-1, 1, [2, 6, 5, 4]).astype(np.float16) From 1ad3701bcb56be0391cd070553869e6909cf6553 Mon Sep 17 00:00:00 2001 From: Netpunk <2327994230@qq.com> Date: Thu, 20 Apr 2023 11:46:46 +0800 Subject: [PATCH 10/11] experiment-3 --- python/paddle/fluid/tests/unittests/test_maxout_op.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/python/paddle/fluid/tests/unittests/test_maxout_op.py b/python/paddle/fluid/tests/unittests/test_maxout_op.py index 66fabe4929180..f9755719d9d5d 100644 --- a/python/paddle/fluid/tests/unittests/test_maxout_op.py +++ b/python/paddle/fluid/tests/unittests/test_maxout_op.py @@ -136,9 +136,6 @@ def test_errors(self): self.assertRaises(ValueError, F.maxout, x_float32, 2, 2) -@unittest.skipIf( - not core.is_compiled_with_cuda(), "core is not compiled with CUDA" -) class TestMaxOutOpFP16(TestMaxOutOp): def set_attrs(self): self.dtype = 'float16' @@ -153,7 +150,9 @@ class TestMaxoutFP16Case2(TestMaxOutOpFP16): def set_attrs(self): self.axis = 3 - +@unittest.skipIf( + not core.is_compiled_with_cuda(), "core is not compiled with CUDA" +) class TestMaxoutStaticAPIFP16(unittest.TestCase): def setUp(self): self.x_np = np.random.uniform(-1, 1, [2, 6, 5, 4]).astype(np.float16) From 5ffcb46ed9e63dcce8331922b83489f56fd6659c Mon Sep 17 00:00:00 2001 From: Netpunk <2327994230@qq.com> Date: Wed, 26 Apr 2023 16:07:42 +0800 Subject: [PATCH 11/11] format code --- python/paddle/fluid/tests/unittests/test_maxout_op.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/paddle/fluid/tests/unittests/test_maxout_op.py b/python/paddle/fluid/tests/unittests/test_maxout_op.py index f9755719d9d5d..b6d339c3aab28 100644 --- a/python/paddle/fluid/tests/unittests/test_maxout_op.py +++ b/python/paddle/fluid/tests/unittests/test_maxout_op.py @@ -150,6 +150,7 @@ class TestMaxoutFP16Case2(TestMaxOutOpFP16): def set_attrs(self): self.axis = 3 + @unittest.skipIf( not core.is_compiled_with_cuda(), "core is not compiled with CUDA" )