From 7002030b6dd1a322de5dcd279bd4004ffe235416 Mon Sep 17 00:00:00 2001 From: Yixuan Qiu Date: Fri, 20 May 2022 19:46:16 +0800 Subject: [PATCH] new functions to return the final gradient information --- CHANGELOG.md | 9 +++++++++ examples/example-rosenbrock-box.cpp | 2 ++ examples/example-rosenbrock.cpp | 2 ++ include/LBFGS.h | 23 ++++++++++++++++++---- include/LBFGSB.h | 30 +++++++++++++++++++++++------ 5 files changed, 56 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f5ab79f4..bc09eaa2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,12 @@ +## Unrealeased + +### Added +- Added functions `final_grad()` and `final_grad_norm()` to `LBFGSSolver` + and `LBFGSBSolver` to retrieve the final gradient information + ([#12](https://github.com/yixuan/LBFGSpp/issues/12)) + + + ## [0.2.0] - 2022-05-20 ### Added diff --git a/examples/example-rosenbrock-box.cpp b/examples/example-rosenbrock-box.cpp index d5d1e53c..f716520d 100644 --- a/examples/example-rosenbrock-box.cpp +++ b/examples/example-rosenbrock-box.cpp @@ -58,6 +58,8 @@ int main() std::cout << niter << " iterations" << std::endl; std::cout << "x = \n" << x.transpose() << std::endl; std::cout << "f(x) = " << fx << std::endl; + std::cout << "grad = " << solver.final_grad().transpose() << std::endl; + std::cout << "projected grad norm = " << solver.final_grad_norm() << std::endl; return 0; } diff --git a/examples/example-rosenbrock.cpp b/examples/example-rosenbrock.cpp index b83a6c95..25089803 100644 --- a/examples/example-rosenbrock.cpp +++ b/examples/example-rosenbrock.cpp @@ -41,6 +41,8 @@ int main() std::cout << niter << " iterations" << std::endl; std::cout << "x = \n" << x.transpose() << std::endl; std::cout << "f(x) = " << fx << std::endl; + std::cout << "grad = " << solver.final_grad().transpose() << std::endl; + std::cout << "||grad|| = " << solver.final_grad_norm() << std::endl; return 0; } diff --git a/include/LBFGS.h b/include/LBFGS.h index 1d29f94d..fd8da744 100644 --- a/include/LBFGS.h +++ b/include/LBFGS.h @@ -30,6 +30,7 @@ class LBFGSSolver Vector m_fx; // History of the objective function values Vector m_xp; // Old x Vector m_grad; // New gradient + Scalar m_gnorm; // Norm of the gradient Vector m_gradp; // Old gradient Vector m_drt; // Moving direction @@ -87,12 +88,12 @@ class LBFGSSolver // Evaluate function and compute gradient fx = f(x, m_grad); - Scalar gnorm = m_grad.norm(); + m_gnorm = m_grad.norm(); if (fpast > 0) m_fx[0] = fx; // Early exit if the initial x is already a minimizer - if (gnorm <= m_param.epsilon || gnorm <= m_param.epsilon_rel * x.norm()) + if (m_gnorm <= m_param.epsilon || m_gnorm <= m_param.epsilon_rel * x.norm()) { return 1; } @@ -114,10 +115,10 @@ class LBFGSSolver LineSearch::LineSearch(f, fx, x, m_grad, step, m_drt, m_xp, m_param); // New gradient norm - gnorm = m_grad.norm(); + m_gnorm = m_grad.norm(); // Convergence test -- gradient - if (gnorm <= m_param.epsilon || gnorm <= m_param.epsilon_rel * x.norm()) + if (m_gnorm <= m_param.epsilon || m_gnorm <= m_param.epsilon_rel * x.norm()) { return k; } @@ -151,6 +152,20 @@ class LBFGSSolver return k; } + + /// + /// Returning the gradient vector on the last iterate. + /// Typically used to debug and test convergence. + /// Should only be called after the `minimize()` function. + /// + /// \return A const reference to the gradient vector. + /// + const Vector& final_grad() const { return m_grad; } + + /// + /// Returning the Euclidean norm of the final gradient. + /// + Scalar final_grad_norm() const { return m_gnorm; } }; } // namespace LBFGSpp diff --git a/include/LBFGSB.h b/include/LBFGSB.h index cf52b04e..675da7a2 100644 --- a/include/LBFGSB.h +++ b/include/LBFGSB.h @@ -33,6 +33,7 @@ class LBFGSBSolver Vector m_fx; // History of the objective function values Vector m_xp; // Old x Vector m_grad; // New gradient + Scalar m_projgnorm; // Projected gradient norm Vector m_gradp; // Old gradient Vector m_drt; // Moving direction @@ -134,15 +135,15 @@ class LBFGSBSolver // Evaluate function and compute gradient fx = f(x, m_grad); - Scalar projgnorm = proj_grad_norm(x, m_grad, lb, ub); + m_projgnorm = proj_grad_norm(x, m_grad, lb, ub); if (fpast > 0) m_fx[0] = fx; // std::cout << "x0 = " << x.transpose() << std::endl; - // std::cout << "f(x0) = " << fx << ", ||proj_grad|| = " << projgnorm << std::endl << std::endl; + // std::cout << "f(x0) = " << fx << ", ||proj_grad|| = " << m_projgnorm << std::endl << std::endl; // Early exit if the initial x is already a minimizer - if (projgnorm <= m_param.epsilon || projgnorm <= m_param.epsilon_rel * x.norm()) + if (m_projgnorm <= m_param.epsilon || m_projgnorm <= m_param.epsilon_rel * x.norm()) { return 1; } @@ -181,14 +182,14 @@ class LBFGSBSolver LineSearch::LineSearch(f, fx, x, m_grad, step, step_max, m_drt, m_xp, m_param); // New projected gradient norm - projgnorm = proj_grad_norm(x, m_grad, lb, ub); + m_projgnorm = proj_grad_norm(x, m_grad, lb, ub); /* std::cout << "** Iteration " << k << std::endl; std::cout << " x = " << x.transpose() << std::endl; - std::cout << " f(x) = " << fx << ", ||proj_grad|| = " << projgnorm << std::endl << std::endl; */ + std::cout << " f(x) = " << fx << ", ||proj_grad|| = " << m_projgnorm << std::endl << std::endl; */ // Convergence test -- gradient - if (projgnorm <= m_param.epsilon || projgnorm <= m_param.epsilon_rel * x.norm()) + if (m_projgnorm <= m_param.epsilon || m_projgnorm <= m_param.epsilon_rel * x.norm()) { return k; } @@ -238,6 +239,23 @@ class LBFGSBSolver return k; } + + /// + /// Returning the gradient vector on the last iterate. + /// Typically used to debug and test convergence. + /// Should only be called after the `minimize()` function. + /// + /// \return A const reference to the gradient vector. + /// + const Vector& final_grad() const { return m_grad; } + + /// + /// Returning the infinity norm of the final projected gradient. + /// The projected gradient is defined as \f$P(x-g,l,u)-x\f$, where \f$P(v,l,u)\f$ stands for + /// the projection of a vector \f$v\f$ onto the box specified by the lower bound vector \f$l\f$ and + /// upper bound vector \f$u\f$. + /// + Scalar final_grad_norm() const { return m_projgnorm; } }; } // namespace LBFGSpp