replace NaturalExpDecay, ExponentialDecay, InverseTimeDecay with 2.0 version

python/paddle/distributed/passes/ps_server_pass.py

@@ -18,7 +18,6 @@
 from paddle.fluid.layers.learning_rate_scheduler import (
     exponential_decay,
     inverse_time_decay,
-    natural_exp_decay,
     noam_decay,
 )
 from paddle.optimizer.lr import (
@@ -112,9 +111,9 @@ def _get_lr_scheduler_program(self, lr_scheduler, lr_decay_steps):
             with paddle.static.program_guard(
                 decay_main_program, decay_startup_program
             ):
-                lr = natural_exp_decay(
-                    1.0, lr_decay_steps, lr_scheduler.gamma, True
-                )
+                lr = paddle.optimizer.lr.NaturalExpDecay(
+                    1.0, lr_scheduler.gamma
+                ).get_lr()
                 lr_name = lr.name
                 logging.warn(
                     "NaturalExpDecay is set, staircase = True, global learning rate decay step is [ %d ], Change decay steps as follow: \n"

python/paddle/fluid/dygraph/learning_rate_scheduler.py

@@ -24,9 +24,6 @@
 __all__ = [
     'NoamDecay',
     'PiecewiseDecay',
-    'NaturalExpDecay',
-    'ExponentialDecay',
-    'InverseTimeDecay',
     'PolynomialDecay',
     'CosineDecay',
     'LinearLrWarmup',
@@ -197,255 +194,6 @@ def step(self):
         return self.create_lr_var(self.vars[len(self.values) - 1])
 
 
-class NaturalExpDecay(LearningRateDecay):
-    r"""
-    :api_attr: imperative
-
-    Applies natural exponential decay to the initial learning rate.
-
-    The algorithm can be described as following.
-
-    .. math::
-
-        decayed\_learning\_rate = learning\_rate * e^{y}
-
-    If staircase is set to False, then:
-
-    .. math::
-
-        y = - decay\_rate * \\frac{global\_step}{decay\_steps}
-
-    If staircase is set to True, then:
-
-    .. math::
-
-        y = - decay\_rate * math.floor(\\frac{global\_step}{decay\_steps})
-
-    Parameters:
-        learning_rate(Variable|float): The initial learning rate. If the type
-            is Variable, it's a tensor with shape [1], the data type can be
-            float32 or float64. It also can be set to python int number.
-        decay_steps(int): The decay step size. It determines the decay cycle.
-        decay_rate(int): The decay rate.
-        staircase(bool, optional): If set to True, decay the learning rate at discrete intervals. The
-            default value is False.
-        begin(int, optional): The begin step. The initial value of global_step described above. The default value is 0.
-        step(int, optional): The step size used to calculate the new global_step in the description above.
-            The default value is 1.
-        dtype(str, optional): The data type used to create the learning rate variable. The data type can be set as
-            'float32', 'float64'. The default value is 'float32'.
-
-    Returns:
-        None.
-
-    Examples:
-        .. code-block:: python
-
-            import paddle.fluid as fluid
-            import paddle
-            base_lr = 0.1
-            with fluid.dygraph.guard():
-                emb = paddle.nn.Embedding(10, 10)
-                sgd_optimizer = fluid.optimizer.SGD(
-                        learning_rate=fluid.dygraph.NaturalExpDecay(
-                            learning_rate=base_lr,
-                            decay_steps=10000,
-                            decay_rate=0.5,
-                            staircase=True),
-                        parameter_list=emb.parameters())
-
-    """
-
-    def __init__(
-        self,
-        learning_rate,
-        decay_steps,
-        decay_rate,
-        staircase=False,
-        begin=0,
-        step=1,
-        dtype='float32',
-    ):
-        super().__init__(begin, step, dtype)
-        self.learning_rate = learning_rate
-        self.decay_steps = decay_steps
-        self.decay_rate = decay_rate
-        self.staircase = staircase
-
-    def step(self):
-        div_res = self.create_lr_var(self.step_num / self.decay_steps)
-        if self.staircase:
-            div_res = paddle.floor(div_res)
-        decayed_lr = self.learning_rate * paddle.exp(
-            -1 * self.decay_rate * div_res
-        )
-
-        return decayed_lr
-
-
-class ExponentialDecay(LearningRateDecay):
-    r"""
-    :api_attr: imperative
-
-    Applies exponential decay to the learning rate.
-
-    The algorithm can be described as following.
-
-    .. math::
-
-        decayed\_learning\_rate = learning\_rate * decay\_rate ^ y
-
-    If staircase is set to False, then:
-
-    .. math::
-
-        y = \\frac{global\_step}{decay\_steps}
-
-    If staircase is set to True, then:
-
-    .. math::
-
-        y = math.floor(\\frac{global\_step}{decay\_steps})
-
-
-    Parameters:
-        learning_rate(Variable|float): The initial learning rate. If the type
-            is Variable, it's a tensor with shape [1], the data type can be
-            float32 or float64. It also can be set to python int number.
-        decay_steps(int): The decay step size. It determines the decay cycle.
-        decay_rate(float): The decay rate.
-        staircase(bool, optional): If set to True, decay the learning rate at discrete intervals. The
-            default value is False.
-        begin(int, optional): The begin step. The initial value of global_step described above. The default value is 0.
-        step(int, optional): The step size used to calculate the new global_step in the description above.
-            The default value is 1.
-        dtype(str, optional): The data type used to create the learning rate variable. The data type can be set as
-            'float32', 'float64'. The default value is 'float32'.
-
-    Returns:
-        None.
-
-    Examples:
-        .. code-block:: python
-
-          import paddle.fluid as fluid
-          base_lr = 0.1
-          with fluid.dygraph.guard():
-              sgd_optimizer = fluid.optimizer.SGD(
-                    learning_rate=fluid.dygraph.ExponentialDecay(
-                        learning_rate=base_lr,
-                        decay_steps=10000,
-                        decay_rate=0.5,
-                        staircase=True))
-
-    """
-
-    def __init__(
-        self,
-        learning_rate,
-        decay_steps,
-        decay_rate,
-        staircase=False,
-        begin=0,
-        step=1,
-        dtype='float32',
-    ):
-        super().__init__(begin, step, dtype)
-        self.learning_rate = learning_rate
-        self.decay_steps = decay_steps
-        self.decay_rate = decay_rate
-        self.staircase = staircase
-
-    def step(self):
-        div_res = self.create_lr_var(self.step_num / self.decay_steps)
-        if self.staircase:
-            div_res = paddle.floor(div_res)
-
-        decayed_lr = self.learning_rate * (self.decay_rate**div_res)
-
-        return decayed_lr
-
-
-class InverseTimeDecay(LearningRateDecay):
-    r"""
-    :api_attr: imperative
-
-    Applies inverse time decay to the initial learning rate.
-
-    The algorithm can be described as following.
-    If staircase is set to False, then:
-
-    .. math::
-
-        decayed\_learning\_rate = \\frac{learning\_rate}{1 + decay\_rate * \\frac{global\_step}{decay\_step}}
-
-    If staircase is set to True, then:
-
-    .. math::
-
-        decayed\_learning\_rate = \\frac{learning\_rate}{1 + decay\_rate * math.floor(\\frac{global\_step}{decay\_step})}
-
-    Parameters:
-        learning_rate(Variable|float): The initial learning rate. If the type
-            is Variable, it's a tensor with shape [1], the data type can be
-            float32 or float64. It also can be set to python int number.
-        decay_steps(int): The decay step size. It determines the decay cycle.
-        decay_rate(float): The decay rate.
-        staircase(bool, optional): If set to True, decay the learning rate at discrete intervals. The
-            default value is False.
-        begin(int, optional): The begin step. The initial value of global_step described above. The default value is 0.
-        step(int, optional): The step size used to calculate the new global_step in the description above.
-            The default value is 1.
-        dtype(str, optional): The data type used to create the learning rate variable. The data type can be
-            'float32', 'float64'. The default value is 'float32'.
-
-    Returns:
-        None.
-
-    Examples:
-        .. code-block:: python
-
-          import paddle.fluid as fluid
-          import paddle
-          base_lr = 0.1
-          with fluid.dygraph.guard():
-              emb = paddle.nn.Embedding(10, 10)
-              sgd_optimizer = fluid.optimizer.SGD(
-                  learning_rate=fluid.dygraph.InverseTimeDecay(
-                        learning_rate=base_lr,
-                        decay_steps=10000,
-                        decay_rate=0.5,
-                        staircase=True),
-                  parameter_list = emb.parameters())
-
-    """
-
-    def __init__(
-        self,
-        learning_rate,
-        decay_steps,
-        decay_rate,
-        staircase=False,
-        begin=0,
-        step=1,
-        dtype='float32',
-    ):
-        super().__init__(begin, step, dtype)
-        self.learning_rate = learning_rate
-        self.decay_steps = decay_steps
-        self.decay_rate = decay_rate
-        self.staircase = staircase
-
-    def step(self):
-        div_res = self.create_lr_var(self.step_num / self.decay_steps)
-        if self.staircase:
-            div_res = paddle.floor(div_res)
-
-        decayed_lr = self.learning_rate / (1 + self.decay_rate * div_res)
-
-        return decayed_lr
-
-
 class PolynomialDecay(LearningRateDecay):
     r"""
     :api_attr: imperative

python/paddle/fluid/layers/learning_rate_scheduler.py

@@ -166,8 +166,8 @@ def exponential_decay(learning_rate, decay_steps, decay_rate, staircase=False):
     """
     with default_main_program()._lr_schedule_guard():
         if in_dygraph_mode():
-            decay = imperate_lr.ExponentialDecay(
-                learning_rate, decay_steps, decay_rate, staircase
+            decay = paddle.optimizer.lr.ExponentialDecay(
+                learning_rate, decay_rate
             )
             return decay
         else:
@@ -228,8 +228,8 @@ def natural_exp_decay(learning_rate, decay_steps, decay_rate, staircase=False):
     """
     with default_main_program()._lr_schedule_guard():
         if in_dygraph_mode():
-            decay = imperate_lr.NaturalExpDecay(
-                learning_rate, decay_steps, decay_rate, staircase
+            decay = paddle.optimizer.lr.NaturalExpDecay(
+                learning_rate, decay_rate
             )
             return decay
         else:
@@ -288,8 +288,8 @@ def inverse_time_decay(learning_rate, decay_steps, decay_rate, staircase=False):
     """
     with default_main_program()._lr_schedule_guard():
         if in_dygraph_mode():
-            decay = imperate_lr.InverseTimeDecay(
-                learning_rate, decay_steps, decay_rate, staircase
+            decay = paddle.optimizer.lr.InverseTimeDecay(
+                learning_rate, decay_rate
             )
             return decay
         else: