From 7de03e403a954ccd89ed0821bbb348fcb422c4c5 Mon Sep 17 00:00:00 2001 From: 0x45f Date: Tue, 14 Sep 2021 07:25:56 +0000 Subject: [PATCH 1/4] fix no_grad context error in dy2stat --- .../dygraph_to_static/partial_program.py | 6 ++++-- .../dygraph_to_static/test_partial_program.py | 21 +++++++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/partial_program.py b/python/paddle/fluid/dygraph/dygraph_to_static/partial_program.py index 663f95af89ce9..c843a72e88e25 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/partial_program.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/partial_program.py @@ -296,7 +296,8 @@ def __call__(self, inputs): @property def program(self): - if self.training: + tracer = framework._dygraph_tracer() + if self.training and tracer._has_grad: return self._train_amp_program if _in_amp_guard( ) else self._train_program else: @@ -304,7 +305,8 @@ def program(self): @property def program_id(self): - if self.training: + tracer = framework._dygraph_tracer() + if self.training and tracer._has_grad: return self._train_amp_program_id if _in_amp_guard( ) else self._train_program_id else: diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_partial_program.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_partial_program.py index 91067f360995e..fd6bcc0141a07 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_partial_program.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_partial_program.py @@ -152,6 +152,27 @@ def test_switch_eval_and_train(self): partial_layer._train_program) +class TestWithNoGrad(unittest.TestCase): + def test_with_no_grad(self): + with fluid.dygraph.guard(): + linear_net = Linear() + x_data = np.random.random((5, 10)).astype('float32') + x = fluid.dygraph.to_variable(x_data) + + linear_net.train() + linear_net(x) + + _, partial_layer = linear_net.forward.program_cache.last()[-1] + # check default mode is for training + self.assertEqual(partial_layer.program, + partial_layer._train_program) + + with paddle.no_grad(): + linear_net(x) + self.assertEqual(partial_layer.program, + partial_layer._infer_program) + + class GPT2LMHeadModel(fluid.dygraph.Layer): def __init__(self): super(GPT2LMHeadModel, self).__init__() From efa655f9a9baf3a5fe675e21c69f22b09778e599 Mon Sep 17 00:00:00 2001 From: 0x45f Date: Tue, 14 Sep 2021 07:31:49 +0000 Subject: [PATCH 2/4] remove useless comments --- .../tests/unittests/dygraph_to_static/test_partial_program.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_partial_program.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_partial_program.py index fd6bcc0141a07..c3ba9d54c7a4a 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_partial_program.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_partial_program.py @@ -163,7 +163,6 @@ def test_with_no_grad(self): linear_net(x) _, partial_layer = linear_net.forward.program_cache.last()[-1] - # check default mode is for training self.assertEqual(partial_layer.program, partial_layer._train_program) From ae647074e14d02184ef0154c439b72d18e505ec2 Mon Sep 17 00:00:00 2001 From: 0x45f Date: Wed, 15 Sep 2021 06:59:41 +0000 Subject: [PATCH 3/4] fix error by drop_kids in python --- paddle/fluid/pybind/pybind.cc | 11 ++++++++++ .../dygraph_to_static/partial_program.py | 13 +++++++----- .../dygraph_to_static/test_partial_program.py | 20 ------------------- 3 files changed, 19 insertions(+), 25 deletions(-) diff --git a/paddle/fluid/pybind/pybind.cc b/paddle/fluid/pybind/pybind.cc index f797ed5142c3d..ef1899e54d685 100644 --- a/paddle/fluid/pybind/pybind.cc +++ b/paddle/fluid/pybind/pybind.cc @@ -1241,6 +1241,17 @@ All parameter, weight, gradient are variables in Paddle. return self.GetMutable(); }, py::return_value_policy::reference) + .def("get_scope", + [](Variable &self) -> Scope * { + auto scope_vec = + self.GetMutable>(); + PADDLE_ENFORCE_GT( + scope_vec->size(), 0, + platform::errors::InvalidArgument( + "The size of scope_vec should greater than 0")); + return scope_vec->front(); + }, + py::return_value_policy::reference) .def("set_scope", [](Variable &self, Scope &scope) { auto scope_vec = self.GetMutable>(); scope_vec->emplace_back(&scope); diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/partial_program.py b/python/paddle/fluid/dygraph/dygraph_to_static/partial_program.py index c843a72e88e25..a1d1fbeacc252 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/partial_program.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/partial_program.py @@ -290,14 +290,18 @@ def __call__(self, inputs): self._valid_vars(self._params), self._valid_vars(out_vars), self._tmp_scope_vec, self._double_grads, *attrs) - + self.drop_kids_in_nograd() restored_nest_out = self._restore_out(out_vars) return self._remove_no_value(restored_nest_out) + def drop_kids_in_nograd(self): + tracer = framework._dygraph_tracer() + if self.training and not tracer._has_grad: + self._tmp_scope_vec.value().get_scope().drop_kids() + @property def program(self): - tracer = framework._dygraph_tracer() - if self.training and tracer._has_grad: + if self.training: return self._train_amp_program if _in_amp_guard( ) else self._train_program else: @@ -305,8 +309,7 @@ def program(self): @property def program_id(self): - tracer = framework._dygraph_tracer() - if self.training and tracer._has_grad: + if self.training: return self._train_amp_program_id if _in_amp_guard( ) else self._train_program_id else: diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_partial_program.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_partial_program.py index c3ba9d54c7a4a..91067f360995e 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_partial_program.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_partial_program.py @@ -152,26 +152,6 @@ def test_switch_eval_and_train(self): partial_layer._train_program) -class TestWithNoGrad(unittest.TestCase): - def test_with_no_grad(self): - with fluid.dygraph.guard(): - linear_net = Linear() - x_data = np.random.random((5, 10)).astype('float32') - x = fluid.dygraph.to_variable(x_data) - - linear_net.train() - linear_net(x) - - _, partial_layer = linear_net.forward.program_cache.last()[-1] - self.assertEqual(partial_layer.program, - partial_layer._train_program) - - with paddle.no_grad(): - linear_net(x) - self.assertEqual(partial_layer.program, - partial_layer._infer_program) - - class GPT2LMHeadModel(fluid.dygraph.Layer): def __init__(self): super(GPT2LMHeadModel, self).__init__() From 8f636cddd7f159ecba5ca2e565764d1613525fa3 Mon Sep 17 00:00:00 2001 From: 0x45f Date: Wed, 15 Sep 2021 11:39:02 +0000 Subject: [PATCH 4/4] add test and fix review --- paddle/fluid/pybind/pybind.cc | 2 +- .../dygraph/dygraph_to_static/partial_program.py | 4 ++-- .../dygraph_to_static/test_partial_program.py | 15 +++++++++++++++ 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/paddle/fluid/pybind/pybind.cc b/paddle/fluid/pybind/pybind.cc index ef1899e54d685..a1a3e6181af78 100644 --- a/paddle/fluid/pybind/pybind.cc +++ b/paddle/fluid/pybind/pybind.cc @@ -1248,7 +1248,7 @@ All parameter, weight, gradient are variables in Paddle. PADDLE_ENFORCE_GT( scope_vec->size(), 0, platform::errors::InvalidArgument( - "The size of scope_vec should greater than 0")); + "The size of scope_vec should be greater than 0")); return scope_vec->front(); }, py::return_value_policy::reference) diff --git a/python/paddle/fluid/dygraph/dygraph_to_static/partial_program.py b/python/paddle/fluid/dygraph/dygraph_to_static/partial_program.py index a1d1fbeacc252..9ccd2321b638a 100644 --- a/python/paddle/fluid/dygraph/dygraph_to_static/partial_program.py +++ b/python/paddle/fluid/dygraph/dygraph_to_static/partial_program.py @@ -290,11 +290,11 @@ def __call__(self, inputs): self._valid_vars(self._params), self._valid_vars(out_vars), self._tmp_scope_vec, self._double_grads, *attrs) - self.drop_kids_in_nograd() + self.drop_scope_if_no_grad() restored_nest_out = self._restore_out(out_vars) return self._remove_no_value(restored_nest_out) - def drop_kids_in_nograd(self): + def drop_scope_if_no_grad(self): tracer = framework._dygraph_tracer() if self.training and not tracer._has_grad: self._tmp_scope_vec.value().get_scope().drop_kids() diff --git a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_partial_program.py b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_partial_program.py index 91067f360995e..220347909f978 100644 --- a/python/paddle/fluid/tests/unittests/dygraph_to_static/test_partial_program.py +++ b/python/paddle/fluid/tests/unittests/dygraph_to_static/test_partial_program.py @@ -152,6 +152,21 @@ def test_switch_eval_and_train(self): partial_layer._train_program) +class TestWithNoGrad(unittest.TestCase): + def test_with_no_grad(self): + with fluid.dygraph.guard(): + linear_net = Linear() + x_data = np.random.random((5, 10)).astype('float32') + x = fluid.dygraph.to_variable(x_data) + + with paddle.no_grad(): + linear_net.train() + linear_net(x) + _, partial_layer = linear_net.forward.program_cache.last()[-1] + self.assertEqual(partial_layer.program, + partial_layer._train_program) + + class GPT2LMHeadModel(fluid.dygraph.Layer): def __init__(self): super(GPT2LMHeadModel, self).__init__()