The L1regParam can be configured through anonymous function, and each component can be penalized differently.

This weighted regularization for L1 will be very useful to exclude the intercept from regularization.

Author: dbtsai

math/src/main/scala/breeze/optimize/FirstOrderMinimizer.scala

@@ -1,7 +1,7 @@
 package breeze.optimize
 
 import breeze.linalg.norm
-import breeze.math.{MutableCoordinateField, MutableFiniteCoordinateField, NormedModule}
+import breeze.math.{MutableEnumeratedCoordinateField, MutableCoordinateField, MutableFiniteCoordinateField, NormedModule}
 import breeze.optimize.FirstOrderMinimizer.ConvergenceReason
 import breeze.stats.distributions.{RandBasis, ThreadLocalRandomGenerator}
 import breeze.util.Implicits._
@@ -216,7 +216,7 @@ object FirstOrderMinimizer {
     }
 
     @deprecated("Use breeze.optimize.minimize(f, init, params) instead.", "0.10")
-    def minimize[T](f: DiffFunction[T], init: T)(implicit space: MutableCoordinateField[T, Double]): T = {
+    def minimize[T](f: DiffFunction[T], init: T)(implicit space: MutableEnumeratedCoordinateField[T, _, Double]): T = {
       this.iterations(f, init).last.x
     }
 
@@ -242,8 +242,8 @@ object FirstOrderMinimizer {
     }
 
     @deprecated("Use breeze.optimize.iterations(f, init, params) instead.", "0.10")
-    def iterations[T](f: DiffFunction[T], init:T)(implicit space: MutableCoordinateField[T, Double]): Iterator[LBFGS[T]#State] = {
-       if(useL1) new OWLQN[T](maxIterations, 5, regularization, tolerance)(space).iterations(f,init)
+    def iterations[T, K](f: DiffFunction[T], init:T)(implicit space: MutableEnumeratedCoordinateField[T, K, Double]): Iterator[LBFGS[T]#State] = {
+       if(useL1) new OWLQN[T, K](maxIterations, 5, regularization, tolerance)(space).iterations(f,init)
       else (new LBFGS[T](maxIterations, 5, tolerance=tolerance)(space)).iterations(DiffFunction.withL2Regularization(f,regularization),init)
     }
   }

math/src/main/scala/breeze/optimize/OWLQN.scala

@@ -14,9 +14,17 @@ import breeze.math._
  *
  * @author dlwh
  */
-class OWLQN[T](maxIter: Int, m: Int,  l1reg: Double=1.0, tolerance: Double = 1E-8)(implicit space: MutableCoordinateField[T, Double]) extends LBFGS[T](maxIter, m, tolerance=tolerance) with SerializableLogging {
+class OWLQN[T, K](maxIter: Int, m: Int, l1reg: K => Double, tolerance: Double)(implicit space: MutableEnumeratedCoordinateField[T, K, Double]) extends LBFGS[T](maxIter, m, tolerance=tolerance) with SerializableLogging {
+
+  def this(maxIter: Int, m: Int, l1reg: K => Double)(implicit space: MutableEnumeratedCoordinateField[T, K, Double]) = this(maxIter, m, l1reg, 1E-8)
+
+  def this(maxIter: Int, m: Int, l1reg: Double, tolerance: Double = 1E-8)(implicit space:MutableEnumeratedCoordinateField[T, K, Double]) = this(maxIter, m, (_: K) => l1reg, tolerance)
+
+  def this(maxIter: Int, m: Int, l1reg: Double)(implicit space: MutableEnumeratedCoordinateField[T, K, Double]) = this(maxIter, m, (_: K) => l1reg, 1E-8)
+
+  def this(maxIter: Int, m: Int)(implicit space: MutableEnumeratedCoordinateField[T, K, Double]) = this(maxIter, m, (_: K) => 1.0, 1E-8)
+
   require(m > 0)
-  require(l1reg >= 0)
 
   import space._
 
@@ -81,18 +89,25 @@ class OWLQN[T](maxIter: Int, m: Int,  l1reg: Double=1.0, tolerance: Double = 1E-
 
   // Adds in the regularization stuff to the gradient
   override protected def adjust(newX: T, newGrad: T, newVal: Double): (Double, T) = {
-    val res = space.zipMapValues.map(newX, newGrad, {case (xv, v) =>
-      xv match {
-        case 0.0 => {
-          val delta_+ = v + l1reg
-          val delta_- = v - l1reg
-          if (delta_- > 0) delta_- else if (delta_+ < 0) delta_+ else 0.0
+    var adjValue = newVal
+    val res = space.zipMapKeyValues.map(newX, newGrad, {case (i, xv, v) =>
+      val l1regValue = l1reg(i)
+      require(l1regValue >= 0.0)
+
+      if(l1regValue == 0.0) {
+        v
+      } else {
+        adjValue += Math.abs(l1regValue * xv)
+        xv match {
+          case 0.0 => {
+            val delta_+ = v + l1regValue
+            val delta_- = v - l1regValue
+            if (delta_- > 0) delta_- else if (delta_+ < 0) delta_+ else 0.0
+          }
+          case _ => v + math.signum(xv) * l1regValue
         }
-
-        case _ => v + math.signum(xv) * l1reg
       }
     })
-    val adjValue = newVal + l1reg * norm(newX, 1.0)
     adjValue -> res
   }
 
@@ -105,26 +120,3 @@ class OWLQN[T](maxIter: Int, m: Int,  l1reg: Double=1.0, tolerance: Double = 1E-
   }
 
 }
-
-
-object OWLQN {
-  def main(args: Array[String]) {
-    val lbfgs = new OWLQN[DenseVector[Double]](100,4)
-
-    def optimizeThis(init: DenseVector[Double]) = {
-      val f = new DiffFunction[DenseVector[Double]] {
-        def calculate(x: DenseVector[Double]) = {
-          (sum((x - 3.0) :^ 2.0),(x * 2.0) - 6.0)
-        }
-      }
-
-      val result = lbfgs.minimize(f,init)
-    }
-
-    //    optimizeThis(Counter(1->1.0,2->2.0,3->3.0))
-    //    optimizeThis(Counter(3-> -2.0,2->3.0,1-> -10.0))
-    //        optimizeThis(DenseVector(1.0,2.0,3.0))
-    optimizeThis(DenseVector( -0.0,0.0, -0.0))
-  }
-}
-

math/src/main/scala/breeze/optimize/OptimizationPackage.scala

@@ -19,7 +19,7 @@ trait IterableOptimizationPackage[Function, Vector, State] extends OptimizationP
 }
 
 object OptimizationPackage {
-  class LBFGSMinimizationPackage[DF, Vector]()(implicit space: MutableCoordinateField[Vector, Double],
+  class LBFGSMinimizationPackage[DF, Vector]()(implicit space: MutableEnumeratedCoordinateField[Vector, _, Double],
                                                     df: DF <:< DiffFunction[Vector]) extends IterableOptimizationPackage[DF, Vector, LBFGS[Vector]#State] {
     def minimize(fn: DF, init: Vector, options: OptimizationOption*):Vector = {
       iterations(fn, init, options:_*).last.x
@@ -91,7 +91,11 @@ trait OptimizationPackageLowPriority {
 
       val wrapped = fn.throughLens[Wrapper]
 
-      val res = options.foldLeft(OptParams())( (a,b) => b apply a).minimize(new CachedDiffFunction(wrapped)(mutaVspace.copy), wrap(init))
+
+      val params: OptParams = options.foldLeft(OptParams())((a, b) => b apply a)
+      require(!params.useL1, "Sorry, we can't use L1 with immutable objects right now...")
+      val lbfgs: LBFGS[Wrapper] = new LBFGS[Wrapper](tolerance = params.tolerance, maxIter = params.maxIterations)
+      val res = lbfgs.minimize(DiffFunction.withL2Regularization(wrapped, params.regularization), wrap(init))
       unwrap(res)
     }
   }