From ca83726a69243348cf8c381e37312d3eefd47f7c Mon Sep 17 00:00:00 2001 From: Adam Osewski Date: Fri, 6 Aug 2021 00:35:12 +0200 Subject: [PATCH 1/3] Small corrections. --- .../paddle/fluid/contrib/mixed_precision/bf16/amp_utils.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/paddle/fluid/contrib/mixed_precision/bf16/amp_utils.py b/python/paddle/fluid/contrib/mixed_precision/bf16/amp_utils.py index 4551947e0fad2..8ee5cb5629ce3 100644 --- a/python/paddle/fluid/contrib/mixed_precision/bf16/amp_utils.py +++ b/python/paddle/fluid/contrib/mixed_precision/bf16/amp_utils.py @@ -128,8 +128,8 @@ def _insert_cast_op(block, op, idx, src_dtype, dest_dtype): continue if out_var.dtype == core.VarDesc.VarType.FP32: out_var.desc.set_dtype(core.VarDesc.VarType.BF16) - if op.has_attr('out_dtype'): - op._set_attr('out_dtype', core.VarDesc.VarType.BF16) + if op.has_attr('out_dtype'): + op._set_attr('out_dtype', core.VarDesc.VarType.BF16) return num_cast_ops @@ -235,7 +235,7 @@ def bf16_guard(): def are_post_ops_bf16(post_ops, keep_fp32_ops): for post_op in post_ops: for op in post_op: - if op.type in keep_fp32_ops: + if op in keep_fp32_ops: return False return True From 89e4d3d84c4457f13c93bf5491c034e6bf1acd30 Mon Sep 17 00:00:00 2001 From: Adam Osewski Date: Fri, 6 Aug 2021 00:37:07 +0200 Subject: [PATCH 2/3] Fix lr for bf16. --- python/paddle/fluid/tests/book/CMakeLists.txt | 2 +- python/paddle/fluid/tests/book/test_fit_a_line.py | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/python/paddle/fluid/tests/book/CMakeLists.txt b/python/paddle/fluid/tests/book/CMakeLists.txt index 09c650f16e2fb..d8cadae068a38 100644 --- a/python/paddle/fluid/tests/book/CMakeLists.txt +++ b/python/paddle/fluid/tests/book/CMakeLists.txt @@ -12,4 +12,4 @@ set_tests_properties(test_image_classification PROPERTIES TIMEOUT 200) set_tests_properties(test_label_semantic_roles PROPERTIES TIMEOUT 240) set_tests_properties(test_machine_translation PROPERTIES TIMEOUT 120) set_tests_properties(test_rnn_encoder_decoder PROPERTIES TIMEOUT 120) -set_tests_properties(test_fit_a_line PROPERTIES TIMEOUT 120) +set_tests_properties(test_fit_a_line PROPERTIES TIMEOUT 140) diff --git a/python/paddle/fluid/tests/book/test_fit_a_line.py b/python/paddle/fluid/tests/book/test_fit_a_line.py index 65542e2096cc2..a533d1b40cf8d 100644 --- a/python/paddle/fluid/tests/book/test_fit_a_line.py +++ b/python/paddle/fluid/tests/book/test_fit_a_line.py @@ -48,7 +48,8 @@ def train(use_cuda, save_dirname, is_local, use_bf16, pure_bf16): cost = fluid.layers.square_error_cost(input=y_predict, label=y) avg_cost = fluid.layers.mean(cost) - sgd_optimizer = fluid.optimizer.SGD(learning_rate=0.001) + lr = 5e-3 if use_bf16 else 1e-3 + sgd_optimizer = fluid.optimizer.SGD(learning_rate=lr) if use_bf16: sgd_optimizer = amp.bf16.decorate_bf16( @@ -83,7 +84,7 @@ def train_loop(main_program): avg_loss_value, = exe.run(main_program, feed=feeder.feed(data), fetch_list=[avg_cost]) - if avg_loss_value[0] < 10.0 or pure_bf16: + if avg_loss_value[0] < 10.0: if save_dirname is not None: paddle.static.save_inference_model(save_dirname, [x], [y_predict], exe) From a843a505ed8b4fea86eb28188d4fd78dc31c5e2f Mon Sep 17 00:00:00 2001 From: Adam Osewski Date: Mon, 23 Aug 2021 12:30:26 +0200 Subject: [PATCH 3/3] Revert some changes. --- python/paddle/fluid/contrib/mixed_precision/bf16/amp_utils.py | 4 ++-- python/paddle/fluid/tests/book/CMakeLists.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/python/paddle/fluid/contrib/mixed_precision/bf16/amp_utils.py b/python/paddle/fluid/contrib/mixed_precision/bf16/amp_utils.py index 8ee5cb5629ce3..0fb86593b2d62 100644 --- a/python/paddle/fluid/contrib/mixed_precision/bf16/amp_utils.py +++ b/python/paddle/fluid/contrib/mixed_precision/bf16/amp_utils.py @@ -128,8 +128,8 @@ def _insert_cast_op(block, op, idx, src_dtype, dest_dtype): continue if out_var.dtype == core.VarDesc.VarType.FP32: out_var.desc.set_dtype(core.VarDesc.VarType.BF16) - if op.has_attr('out_dtype'): - op._set_attr('out_dtype', core.VarDesc.VarType.BF16) + if op.has_attr('out_dtype'): + op._set_attr('out_dtype', core.VarDesc.VarType.BF16) return num_cast_ops diff --git a/python/paddle/fluid/tests/book/CMakeLists.txt b/python/paddle/fluid/tests/book/CMakeLists.txt index d8cadae068a38..09c650f16e2fb 100644 --- a/python/paddle/fluid/tests/book/CMakeLists.txt +++ b/python/paddle/fluid/tests/book/CMakeLists.txt @@ -12,4 +12,4 @@ set_tests_properties(test_image_classification PROPERTIES TIMEOUT 200) set_tests_properties(test_label_semantic_roles PROPERTIES TIMEOUT 240) set_tests_properties(test_machine_translation PROPERTIES TIMEOUT 120) set_tests_properties(test_rnn_encoder_decoder PROPERTIES TIMEOUT 120) -set_tests_properties(test_fit_a_line PROPERTIES TIMEOUT 140) +set_tests_properties(test_fit_a_line PROPERTIES TIMEOUT 120)