Fix incorrect computation of accum grad

param decay should be accumulated after applying learning rate.
tnarihi · Mar 12, 2015 · a01dfbf · a01dfbf
1 parent 7197702
commit a01dfbf
Showing 1 changed file with 4 additions and 2 deletions.
diff --git a/src/caffe/solver.cpp b/src/caffe/solver.cpp
@@ -498,7 +498,8 @@ void SGDSolver<Dtype>::ComputeUpdateValue() {
  // Compute the value to history, and then copy them to the blob's diff.
  Dtype local_rate = rate * net_params_lr[param_id]
  / this->param_.iter_size();
- Dtype local_decay = weight_decay * net_params_weight_decay[param_id];
+ Dtype local_decay = weight_decay * net_params_weight_decay[param_id]
+ * this->param_.iter_size();
 
  if (local_decay) {
  if (regularization_type == "L2") {
@@ -535,7 +536,8 @@ void SGDSolver<Dtype>::ComputeUpdateValue() {
  // Compute the value to history, and then copy them to the blob's diff.
  Dtype local_rate = rate * net_params_lr[param_id]
  / this->param_.iter_size();
- Dtype local_decay = weight_decay * net_params_weight_decay[param_id];
+ Dtype local_decay = weight_decay * net_params_weight_decay[param_id]
+ * this->param_.iter_size();
 
  if (local_decay) {
  if (regularization_type == "L2") {