From 05064057f4bbaf6ed9ab18ee8910fac4cf0962b8 Mon Sep 17 00:00:00 2001 From: Asthestarsfalll <1186454801@qq.com> Date: Mon, 25 Sep 2023 21:21:18 +0800 Subject: [PATCH 1/7] Add LinearLR --- python/paddle/optimizer/lr.py | 100 ++++++++++++++++++++++++++ test/legacy_test/test_lr_scheduler.py | 59 +++++++++++++++ 2 files changed, 159 insertions(+) diff --git a/python/paddle/optimizer/lr.py b/python/paddle/optimizer/lr.py index 6fb777447f8a1..11b9f43afa417 100644 --- a/python/paddle/optimizer/lr.py +++ b/python/paddle/optimizer/lr.py @@ -45,6 +45,7 @@ 'MultiplicativeDecay', 'OneCycleLR', 'CyclicLR', + 'LinearLR', ] @@ -2229,6 +2230,105 @@ def get_lr(self): return lr +class LinearLR(LRScheduler): + r""" + Set the learning rate according to linear scheduler. + The learning rate will be firstly multiplied by start_factor and linearly increase to end learning rate. + + Args: + learning_rate (float): The initial learning rate. It is a python float number. + total_steps (int): Number of iterations that the learning_rate reaches end learning_rate. + start_factor (float): Start learning rate is defined by `start_factor * learning_rate` . Default: 1./3. + end_factor (float) End learning rate is defined by `end_factor * learning_rate`. Default: 1.0. + last_epoch (int, optional): The index of last epoch. Can be set to restart training.Default: -1, means initial learning rate. + verbose: (bool, optional): If ``True``, prints a message to stdout for each update. Default: ``False`` . + + Returns: + ``LinearLR`` instance to schedule learning rate. + + Examples: + .. code-block:: python + :name: code-example1 + + >>> # Example1: train on default dynamic graph mode + >>> import paddle + >>> import numpy as np + + >>> # train on default dynamic graph mode + >>> linear = paddle.nn.Linear(10, 10) + >>> scheduler = paddle.optimizer.lr.LinearLR(learning_rate=0.5, total_steps=5, verbose=True) + >>> sgd = paddle.optimizer.SGD(learning_rate=scheduler, parameters=linear.parameters()) + >>> for epoch in range(5): + ... for batch_id in range(20): + ... x = paddle.uniform([10, 10]) + ... out = linear(x) + ... loss = paddle.mean(out) + ... loss.backward() + ... sgd.step() + ... sgd.clear_gradients() + ... scheduler.step() + + .. code-block:: python + :name: code-example2 + + >>> # Example2: train on static graph mode + >>> import paddle + >>> import numpy as np + >>> paddle.enable_static() + >>> main_prog = paddle.static.Program() + >>> start_prog = paddle.static.Program() + >>> with paddle.static.program_guard(main_prog, start_prog): + ... x = paddle.static.data(name='x', shape=[None, 4, 5]) + ... y = paddle.static.data(name='y', shape=[None, 4, 5]) + ... z = paddle.static.nn.fc(x, 100) + ... loss = paddle.mean(z) + ... scheduler = paddle.optimizer.lr.LinearLR(learning_rate=0.5, + ... total_steps=5, verbose=True) + ... sgd = paddle.optimizer.SGD(learning_rate=scheduler) + ... sgd.minimize(loss) + ... + >>> exe = paddle.static.Executor() + >>> exe.run(start_prog) + >>> for epoch in range(5): + ... for batch_id in range(20): + ... out = exe.run( + ... main_prog, + ... feed={ + ... 'x': np.random.randn(3, 4, 5).astype('float32'), + ... 'y': np.random.randn(3, 4, 5).astype('float32') + ... }, + ... fetch_list=loss.name) + ... scheduler.step() + """ + + def __init__(self, learning_rate, total_steps, start_factor=1./3, end_factor=1.0, last_epoch=-1, verbose=False): + if start_factor > 1.0 or start_factor <= 0: + raise ValueError("`start_factor` must be greater than 0 and less or equal to 1, but got {}".format(start_factor)) + + if end_factor > 1.0 or end_factor < 0: + raise ValueError("`end_factor` must be greater than 0 and less than 1, but got {}".format(end_factor)) + + if total_steps <= 0: + raise ValueError("`total_steps` must be greater than 0, but got {}".format(total_steps)) + + self.start_factor = start_factor + self.end_factor = end_factor + self.total_steps = total_steps + + super().__init__(learning_rate, last_epoch, verbose) + + def get_lr(self): + if self.last_epoch == 0: + return self.base_lr * self.start_factor + elif self.last_epoch > self.total_steps: + return self.last_lr + else: + base_lr = self.total_steps * self.start_factor + cur_factor = self.end_factor - self.start_factor + factor = 1. + cur_factor / (base_lr + (self.last_epoch - 1) * cur_factor) + + return self.last_lr * factor + def autoincreased_step_counter(counter_name=None, begin=1, step=1): """ :api_attr: Static Graph diff --git a/test/legacy_test/test_lr_scheduler.py b/test/legacy_test/test_lr_scheduler.py index 54484ecc6ad2c..e83bc7a49e117 100644 --- a/test/legacy_test/test_lr_scheduler.py +++ b/test/legacy_test/test_lr_scheduler.py @@ -463,6 +463,30 @@ def exp_range(x): return base_learning_rate + base_height * scale_fn(eval(scale_mode)) +linear_last_lr = None + +def linear_lr( + epoch_num, + learning_rate, + total_steps, + start_factor=1.0/3, + end_factor=1.0, + verbose=False, +): + global linear_last_lr + if epoch_num == 0: + linear_last_lr = learning_rate * start_factor + return linear_last_lr + elif epoch_num > total_steps: + return linear_last_lr + else: + base_lr = total_steps * start_factor + cur_factor = end_factor - start_factor + factor = 1. + cur_factor / (base_lr + (epoch_num - 1) * cur_factor) + + linear_last_lr *= factor + return linear_last_lr + class TestLRScheduler(unittest.TestCase): def _test_static(self, python_func, paddle_api, kwarg, place): @@ -711,6 +735,21 @@ def test_scheduler(self): paddle.optimizer.lr.PiecewiseDecay( boundaries=[100, 200], values=[0.5, 0.1] ) + # check minus total_steps + with self.assertRaises(ValueError): + paddle.optimizer.lr.LinearLR( + total_steps=-1 + ) + # check start_factor + with self.assertRaises(ValueError): + paddle.optimizer.lr.LinearLR( + total_steps=5, start_factor=2 + ) + # check end_factor + with self.assertRaises(ValueError): + paddle.optimizer.lr.LinearLR( + total_steps=5, end_factor=2 + ) func_api_kwargs = [ ( @@ -944,6 +983,26 @@ def test_scheduler(self): "verbose": False, }, ), + ( + linear_lr, + paddle.optimizer.lr.LinearLR, + { + "total_steps:": 40, + "start_factor": 0.5, + "end_factor": 1, + "verbose": False, + }, + ), + ( + linear_lr, + paddle.optimizer.lr.LinearLR, + { + "total_steps:": 5, + "start_factor": 0.2, + "end_factor": 3, + "verbose": False, + }, + ) ] for python_func, paddle_api, kwarg in func_api_kwargs: From b05538371a0d21cee2f3df5f21053d07f976eaba Mon Sep 17 00:00:00 2001 From: Asthestarsfalll <1186454801@qq.com> Date: Mon, 25 Sep 2023 21:41:40 +0800 Subject: [PATCH 2/7] fix --- python/paddle/optimizer/lr.py | 36 +++++++++++++++++++++------ test/legacy_test/test_lr_scheduler.py | 22 +++++++--------- 2 files changed, 37 insertions(+), 21 deletions(-) diff --git a/python/paddle/optimizer/lr.py b/python/paddle/optimizer/lr.py index 11b9f43afa417..06529c3e722ad 100644 --- a/python/paddle/optimizer/lr.py +++ b/python/paddle/optimizer/lr.py @@ -2301,15 +2301,33 @@ class LinearLR(LRScheduler): ... scheduler.step() """ - def __init__(self, learning_rate, total_steps, start_factor=1./3, end_factor=1.0, last_epoch=-1, verbose=False): - if start_factor > 1.0 or start_factor <= 0: - raise ValueError("`start_factor` must be greater than 0 and less or equal to 1, but got {}".format(start_factor)) + def __init__( + self, + learning_rate, + total_steps, + start_factor=1.0 / 3, + end_factor=1.0, + last_epoch=-1, + verbose=False, + ): + if start_factor > 1.0 or start_factor <= 0: + raise ValueError( + "`start_factor` must be greater than 0 and less or equal to 1, but got {}".format( + start_factor + ) + ) - if end_factor > 1.0 or end_factor < 0: - raise ValueError("`end_factor` must be greater than 0 and less than 1, but got {}".format(end_factor)) + if end_factor > 1.0 or end_factor < 0: + raise ValueError( + "`end_factor` must be greater than 0 and less than 1, but got {}".format( + end_factor + ) + ) if total_steps <= 0: - raise ValueError("`total_steps` must be greater than 0, but got {}".format(total_steps)) + raise ValueError( + f"`total_steps` must be greater than 0, but got {total_steps}" + ) self.start_factor = start_factor self.end_factor = end_factor @@ -2325,10 +2343,12 @@ def get_lr(self): else: base_lr = self.total_steps * self.start_factor cur_factor = self.end_factor - self.start_factor - factor = 1. + cur_factor / (base_lr + (self.last_epoch - 1) * cur_factor) - + factor = 1.0 + cur_factor / ( + base_lr + (self.last_epoch - 1) * cur_factor + ) return self.last_lr * factor + def autoincreased_step_counter(counter_name=None, begin=1, step=1): """ :api_attr: Static Graph diff --git a/test/legacy_test/test_lr_scheduler.py b/test/legacy_test/test_lr_scheduler.py index e83bc7a49e117..030ff31de4e3a 100644 --- a/test/legacy_test/test_lr_scheduler.py +++ b/test/legacy_test/test_lr_scheduler.py @@ -463,13 +463,15 @@ def exp_range(x): return base_learning_rate + base_height * scale_fn(eval(scale_mode)) + linear_last_lr = None + def linear_lr( epoch_num, learning_rate, total_steps, - start_factor=1.0/3, + start_factor=1.0 / 3, end_factor=1.0, verbose=False, ): @@ -482,7 +484,7 @@ def linear_lr( else: base_lr = total_steps * start_factor cur_factor = end_factor - start_factor - factor = 1. + cur_factor / (base_lr + (epoch_num - 1) * cur_factor) + factor = 1.0 + cur_factor / (base_lr + (epoch_num - 1) * cur_factor) linear_last_lr *= factor return linear_last_lr @@ -737,19 +739,13 @@ def test_scheduler(self): ) # check minus total_steps with self.assertRaises(ValueError): - paddle.optimizer.lr.LinearLR( - total_steps=-1 - ) + paddle.optimizer.lr.LinearLR(total_steps=-1) # check start_factor with self.assertRaises(ValueError): - paddle.optimizer.lr.LinearLR( - total_steps=5, start_factor=2 - ) + paddle.optimizer.lr.LinearLR(total_steps=5, start_factor=2) # check end_factor with self.assertRaises(ValueError): - paddle.optimizer.lr.LinearLR( - total_steps=5, end_factor=2 - ) + paddle.optimizer.lr.LinearLR(total_steps=5, end_factor=2) func_api_kwargs = [ ( @@ -999,10 +995,10 @@ def test_scheduler(self): { "total_steps:": 5, "start_factor": 0.2, - "end_factor": 3, + "end_factor": 0.5, "verbose": False, }, - ) + ), ] for python_func, paddle_api, kwarg in func_api_kwargs: From 44469ead9b4145d506d7a60e34d87a274e92ab43 Mon Sep 17 00:00:00 2001 From: Asthestarsfalll <1186454801@qq.com> Date: Tue, 26 Sep 2023 00:21:15 +0800 Subject: [PATCH 3/7] fix unittest --- test/legacy_test/test_lr_scheduler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/legacy_test/test_lr_scheduler.py b/test/legacy_test/test_lr_scheduler.py index 030ff31de4e3a..e5b137b192ffe 100644 --- a/test/legacy_test/test_lr_scheduler.py +++ b/test/legacy_test/test_lr_scheduler.py @@ -485,7 +485,6 @@ def linear_lr( base_lr = total_steps * start_factor cur_factor = end_factor - start_factor factor = 1.0 + cur_factor / (base_lr + (epoch_num - 1) * cur_factor) - linear_last_lr *= factor return linear_last_lr @@ -983,6 +982,7 @@ def test_scheduler(self): linear_lr, paddle.optimizer.lr.LinearLR, { + "learning_rate": 0.2, "total_steps:": 40, "start_factor": 0.5, "end_factor": 1, @@ -993,6 +993,7 @@ def test_scheduler(self): linear_lr, paddle.optimizer.lr.LinearLR, { + "learning_rate": 0.2, "total_steps:": 5, "start_factor": 0.2, "end_factor": 0.5, From ddd8fc491edc03e3249d76a41cf2cfb8d74e29a1 Mon Sep 17 00:00:00 2001 From: Asthestarsfalll <1186454801@qq.com> Date: Tue, 26 Sep 2023 01:20:28 +0800 Subject: [PATCH 4/7] fix unittest --- test/legacy_test/test_lr_scheduler.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/test/legacy_test/test_lr_scheduler.py b/test/legacy_test/test_lr_scheduler.py index e5b137b192ffe..2716acf6f94f3 100644 --- a/test/legacy_test/test_lr_scheduler.py +++ b/test/legacy_test/test_lr_scheduler.py @@ -738,13 +738,17 @@ def test_scheduler(self): ) # check minus total_steps with self.assertRaises(ValueError): - paddle.optimizer.lr.LinearLR(total_steps=-1) + paddle.optimizer.lr.LinearLR(learning_rate=1, total_steps=-1) # check start_factor with self.assertRaises(ValueError): - paddle.optimizer.lr.LinearLR(total_steps=5, start_factor=2) + paddle.optimizer.lr.LinearLR( + learning_rate=1, total_steps=5, start_factor=2 + ) # check end_factor with self.assertRaises(ValueError): - paddle.optimizer.lr.LinearLR(total_steps=5, end_factor=2) + paddle.optimizer.lr.LinearLR( + learning_rate=1, total_steps=5, end_factor=2 + ) func_api_kwargs = [ ( From 8af28f6f73e198f08155273ede26cd69543042cb Mon Sep 17 00:00:00 2001 From: Asthestarsfalll <1186454801@qq.com> Date: Tue, 26 Sep 2023 02:02:40 +0800 Subject: [PATCH 5/7] fix unittest --- test/legacy_test/test_lr_scheduler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/test/legacy_test/test_lr_scheduler.py b/test/legacy_test/test_lr_scheduler.py index 2716acf6f94f3..ba1f712dce2fd 100644 --- a/test/legacy_test/test_lr_scheduler.py +++ b/test/legacy_test/test_lr_scheduler.py @@ -987,7 +987,7 @@ def test_scheduler(self): paddle.optimizer.lr.LinearLR, { "learning_rate": 0.2, - "total_steps:": 40, + "total_steps": 40, "start_factor": 0.5, "end_factor": 1, "verbose": False, @@ -998,7 +998,7 @@ def test_scheduler(self): paddle.optimizer.lr.LinearLR, { "learning_rate": 0.2, - "total_steps:": 5, + "total_steps": 5, "start_factor": 0.2, "end_factor": 0.5, "verbose": False, From cc79df3b5dda1f39e433ff37fe48a253790a08a0 Mon Sep 17 00:00:00 2001 From: Asthestarsfalll <72954905+Asthestarsfalll@users.noreply.github.com> Date: Thu, 12 Oct 2023 16:46:07 +0800 Subject: [PATCH 6/7] update name Co-authored-by: zachary sun <70642955+sunzhongkai588@users.noreply.github.com> --- python/paddle/optimizer/lr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/optimizer/lr.py b/python/paddle/optimizer/lr.py index 49892b8c659bc..c742bb77499b3 100644 --- a/python/paddle/optimizer/lr.py +++ b/python/paddle/optimizer/lr.py @@ -2248,7 +2248,7 @@ class LinearLR(LRScheduler): Examples: .. code-block:: python - :name: code-example1 + :name: code-dynamic >>> # Example1: train on default dynamic graph mode >>> import paddle From 5f3c2d4a3f9a1df4a629fd8769f1bdb205d07591 Mon Sep 17 00:00:00 2001 From: Asthestarsfalll <72954905+Asthestarsfalll@users.noreply.github.com> Date: Thu, 12 Oct 2023 16:46:34 +0800 Subject: [PATCH 7/7] update name Co-authored-by: zachary sun <70642955+sunzhongkai588@users.noreply.github.com> --- python/paddle/optimizer/lr.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/optimizer/lr.py b/python/paddle/optimizer/lr.py index c742bb77499b3..37a46f53707f1 100644 --- a/python/paddle/optimizer/lr.py +++ b/python/paddle/optimizer/lr.py @@ -2269,7 +2269,7 @@ class LinearLR(LRScheduler): ... scheduler.step() .. code-block:: python - :name: code-example2 + :name: code-static >>> # Example2: train on static graph mode >>> import paddle