"template specialliaztion"

paddle/go/optimizer/CMakeLists.txt

@@ -18,6 +18,8 @@ set(OPITMIZER_Headers
 add_library(optimizer STATIC ${OPITMIZER_SRCS})
 add_dependencies(optimizer gen_proto_cpp)
 
+include_directories("/home/work/dongzhihong/github/Paddle/Paddle/third_party/install/glog/include")
+link_directories("/home/work/dongzhihong/github/Paddle/Paddle/third_party/install/glog/lib")
 add_executable(optimizer_test optimizer_test.cpp)
 add_dependencies(optimizer_test gen_proto_cpp gtest)
 add_test(NAME optimizer_test COMMAND optimzier_test)

paddle/go/optimizer/optimizer_factory.cc

@@ -9,6 +9,8 @@ SGDOptimizer<T>::SGDOptimizer(const OptimizerConfig &config) : ParameterOptimize
  momentum = config.momentum;
  decay = config.decay;
  nesterov = config.nesterov;
+ lr_decay_a = config.lr_decay_a;
+ lr_decay_b = config.lr_decay_b;
 }
 
 template<class T>
@@ -28,11 +30,12 @@ void SGDOptimizer<T>::set_weight(const Tensor<T> *p) {
 
 template<class T>
 void SGDOptimizer<T>::update(const Tensor<T> &gradient) {
- learning_rate = applyLinearLearningRate(config_);
+ num_sample_passed += 1;
+ learning_rate = get_learning_rate();
  for(size_t i=0; i<parameter_.size(); ++i) {
  momentums_[i] = momentum * momentums_[i] - learning_rate*gradient[i] - decay*parameter_[i];
  if(nesterov) {
- //TODO(zhihong) : fix nesterov
+ //TODO(zhihong) : fix nesterov updating
  parameter_[i] += momentums_[i];
  } else {
  parameter_[i] += momentums_[i];
@@ -50,6 +53,7 @@ char* SGDOptimizer<T>::get_config_proto() {
  return config.SerializeAsString().c_str();
 }
 
+
 template class SGDOptimizer<float>;
 template class SGDOptimizer<double>;
 template class AdagradOptimizer<float>;
@@ -58,25 +62,5 @@ template class AdadeltaOptimizer<float>;
 template class AdadeltaOptimizer<double>;
 template class AdamOptimizer<float>;
 template class AdamOptimizer<double>;
-
-
-
-// template<class T>
-// AdagradOptimizer(OptimizerConfig &config) : ParameterOptimizer(config) {
-// learning_rate = config.learning_rate;
-// momentum = config.momentum;
-// decay = config.decay;
-// nesterov = confg.nesterov;
-// }
-
-
-
-
-// template <class T>
-// MomentumOptimizer<T>::MomentumOptimizer(const paddle::optimizer_config &config)
-// : ParameterOptimizer(config) {
-// momentum = config.mometum;
-// }
-
 } // namespace optimizer
 } // namespace paddle

paddle/go/optimizer/optimizer_factory.h

@@ -6,32 +6,6 @@
 namespace paddle {
 namespace optimizer {
 
-static double applyLinearLearningRate(paddle::optimizer_config &config) {
- config.learning_rate =
- std::max(config.learning_rate - config.decay_a * config.samples_processed,
- config.decay_b);
- return config.learning_rate;
-}
-
-static double applyExpLearningRate(paddle::optimizer_config &config) {
- double decayRatio = (double)config.samples_processed / config.decay_b;
- config.learning_rate = config.learning_rate * std::pow(decay_a, decayRatio);
-}
-
-// double applyLearningRate(double learning_rate, uint32_t epoch);
-
-
-// double applyLinearLearningRate(double learning_rate, double decay_a, double
-// decay_b, uint64_t samples_processed);
-
-// double applyExpLearningRate(double learning_rate, double decay_a, double
-// decay_b, uint64_t samples_processed);
-
-// double applyPolyLearningRate(double learning_rate, double decay_a, double
-// decay_b, uint64_t samples_processed, uint32_t epoch);
-
-// double applyCaffePolyLearningRate(double learning_rate, double decay_a,
-// double decay_b, uint64_t samples_processed, uint32_t epoch);
 
 template <class T>
 class SGDOptimizer : public ParameterOptimizer<T> {
@@ -53,6 +27,8 @@ class SGDOptimizer : public ParameterOptimizer<T> {
  double momentum;
  double decay;
  bool nesterov;
+ double lr_decay_a;
+ double lr_decay_b;
 };
 
 template <class T>
@@ -61,33 +37,32 @@ class AdagradOptimizer : public ParameterOptimizer<T> {
  void update(const Tensor<T> &gradient) {
  }
 private:
-
 };
 
 template <class T>
 class AdadeltaOptimizer : public ParameterOptimizer<T> {
 public:
  /*! \brief call the applySGD for example */
  void update(const Tensor<T> &gradient) {
- auto parameter = &(*parameter_.get());
- learning_rate = applyLinearLearningRate(config_);
- applyGradientDescent(parameter, gradient, learning_rate);
  }
 private:
-
+ double learning_rate;
+ double rho;
+ double epsilon;
+ double decay;
 };
 
 template <class T>
 class AdamOptimizer : public ParameterOptimizer<T> {
 public:
  /*! \brief call the applySGD for example */
  void update(const Tensor<T> &gradient) {
- auto parameter = &(*parameter_.get());
- learning_rate = applyLinearLearningRate(config_);
- applyGradientDescent(parameter, gradient, learning_rate);
  }
 private:
-
+ double learning_rate ;
+ double beta_1;
+ double beta_2;
+ double epsilon;
 };
 
 // template <class T>

paddle/go/optimizer/parameter_optimizer.cc

@@ -25,7 +25,14 @@ ParameterOptimizer<T>* ParameterOptimizer<T>::create(
 
 template<class T>
 double ParameterOptimzier<T>::get_learning_rate() {
-
+ if (config_.lr_type == paddle::OptimizerConfig_LearningRateType_Linear) {
+ learning_rate = std::max(learning_rate - lr_decay_a * num_sample_passed, lr_decay_b);
+
+ } else if (config_.lr_type == paddle::OptimizerConfig_LearningRateType_Exp) {
+ double decayRatio = (double)num_sample_passed / lr_decay_b;
+ learning_rate = learning_rate * std::pow(lr_decay_a, decayRatio);
+ }
+ return learning_rate;
 }
 
 template <class T>

paddle/go/optimizer/parameter_optimizer.h

@@ -1,9 +1,9 @@
-#ifndef PADDLE_LIB_OPTIMIZER_BASE_H_
-#define PADDLE_LIB_OPTIMIZER_BASE_H_
+#ifndef PADDLE_PARAMETER_OPTIMIZER_H_
+#define PADDLE_PARAMETER_OPTIMIZER_H_
 
+#include <glog/logging.h>
 #include <functional>
 #include <string>
-// #include <math/Tensor.h>
 #include "Tensor.h"
 #include "OptimizerConfig.pb.h"