From e3471da310372499cbe5dc677dc17646bf2c7372 Mon Sep 17 00:00:00 2001 From: Deniz Oktay Date: Wed, 20 Aug 2014 16:56:40 -0700 Subject: [PATCH 1/9] added COBYLA --- moe/optimal_learning/python/constant.py | 1 + .../python/python_version/domain.py | 10 +- .../python/python_version/optimization.py | 97 +++++++++++++++++++ .../python/repeated_domain.py | 7 ++ .../python_version/optimization_test.py | 73 +++++++++++++- 5 files changed, 183 insertions(+), 5 deletions(-) diff --git a/moe/optimal_learning/python/constant.py b/moe/optimal_learning/python/constant.py index a40192f9..71d8c518 100644 --- a/moe/optimal_learning/python/constant.py +++ b/moe/optimal_learning/python/constant.py @@ -37,6 +37,7 @@ # Domain constants TENSOR_PRODUCT_DOMAIN_TYPE = 'tensor_product' SIMPLEX_INTERSECT_TENSOR_PRODUCT_DOMAIN_TYPE = 'simplex_intersect_tensor_product' +LINEAR_CONSTRAINT_DOMAIN_TYPE = 'linear_constraints' #: Domain types supported by :mod:`moe` DOMAIN_TYPES = [ diff --git a/moe/optimal_learning/python/python_version/domain.py b/moe/optimal_learning/python/python_version/domain.py index 8e7a5782..04058a05 100644 --- a/moe/optimal_learning/python/python_version/domain.py +++ b/moe/optimal_learning/python/python_version/domain.py @@ -17,7 +17,7 @@ import numpy -from moe.optimal_learning.python.constant import TENSOR_PRODUCT_DOMAIN_TYPE +from moe.optimal_learning.python.constant import TENSOR_PRODUCT_DOMAIN_TYPE, LINEAR_CONSTRAINT_DOMAIN_TYPE from moe.optimal_learning.python.geometry_utils import generate_grid_points, generate_latin_hypercube_points from moe.optimal_learning.python.interfaces.domain_interface import DomainInterface @@ -85,6 +85,14 @@ def get_bounding_box(self): """Return a list of ClosedIntervals representing a bounding box for this domain.""" return copy.copy(self._domain_bounds) + def get_constraint_list(self, start_index=0): + """Return a list of lambda functions expressing the domain bounds as linear constraints. Used by COBYLA.""" + constraints = [] + for i, interval in enumerate(self._domain_bounds): + constraints.append((lambda x: x[i + start_index] - interval.min)) + constraints.append((lambda x: interval.max - x[i + start_index])) + return constraints + def generate_random_point_in_domain(self, random_source=None): """Generate ``point`` uniformly at random such that ``self.check_point_inside(point)`` is True. diff --git a/moe/optimal_learning/python/python_version/optimization.py b/moe/optimal_learning/python/python_version/optimization.py index ef9e6429..f1a9b9e1 100644 --- a/moe/optimal_learning/python/python_version/optimization.py +++ b/moe/optimal_learning/python/python_version/optimization.py @@ -324,6 +324,31 @@ class LBFGSBParameters(_BaseLBFGSBParameters): __slots__ = () +# See ConstrainedDFOParameters (below) for docstring. +_BaseConstrainedDFOParameters = collections.namedtuple('_BaseConstrainedDFOParameters', [ + 'rhobeg', + 'rhoend', + 'maxfun', + 'catol', +]) + + +class ConstrainedDFOParameters(_BaseConstrainedDFOParameters): + + r"""Container to hold parameters that specify the behavior of COBYLA. + + Suggested values come from scipy documentation for scipy.optimize.fmin_cobyla. + + :ivar rhobeg: (*float64 > 0.0*) reasonable initial changes to the variables (suggest: 1.0) + :ivar rhoend: (*float64 > 0.0*) final accuracy in the optimization (not precisely guaranteed), which is a lower bound on the size of the trust region (suggest: 1.0e-4) + :ivar maxfun: (*int > 0*) maximum number of objective function calls to make (suggest: 1000) + :ivar catol: (*float64 > 0.0*) absolute tolerance for constraint violations (suggest: 2.0e-4) + + """ + + __slots__ = () + + class NullOptimizer(OptimizerInterface): """A "null" or identity optimizer: this does nothing. It is used to perform "dumb" search with MultistartOptimizer.""" @@ -646,3 +671,75 @@ def optimize(self, **kwargs): else: shaped_point = unshaped_point.reshape(self._num_points, self.domain.dim) self.objective_function.current_point = shaped_point + + +class ConstrainedDFOOptimizer(OptimizerInterface): + + r"""Optimizes an objective function over the specified contraints with the COBYLA method. + + .. Note:: See optimize() docstring for more details. + + """ + + def __init__(self, domain, optimizable, optimization_parameters, num_random_samples=None): + """Construct a ConstrainedDFOOptimizer. + + :param domain: the domain that this optimizer operates over + :type domain: interfaces.domain_interface.DomainInterface subclass. Only supports TensorProductDomain for now. + :param optimizable: object representing the objective function being optimized + :type optimizable: interfaces.optimization_interface.OptimizableInterface subclass + :param optimization_parameters: parameters describing how to perform optimization (tolerances, iterations, etc.) + :type optimization_parameters: python_version.optimization.ConstrainedDFOParameters object + + """ + self.domain = domain + self.objective_function = optimizable + self.optimization_parameters = optimization_parameters + self._num_points = 1 + if hasattr(self.domain, 'num_repeats'): + self._num_points = self.domain.num_repeats + + def _scipy_decorator(self, func, **kwargs): + """Wrapper function for expected improvement calculation to feed into COBYLA. + + func should be of the form compute_* in interfaces.optimization_interface.OptimizableInterface. + """ + def decorated(point): + """Decorator for compute_* functions in interfaces.optimization_interface.OptimizableInterface. + + Converts the point to proper format and sets the current point before calling the compute function. + + :param point: the point on which to do the calculation + :type point: array of float64 with shape (self._num_points * self.domain.dim) + """ + shaped_point = point.reshape(self._num_points, self.domain.dim) + self.objective_function.current_point = shaped_point + value = -func(**kwargs) + if isinstance(value, (numpy.ndarray)): + return value.flatten() + else: + return value + + return decorated + + def optimize(self, **kwargs): + """Perform a COBYLA optimization given the parameters in optimization_parameters. + + objective_function.current_point will be set to the optimal point found. + """ + # Parameters defined above in LBFGSBParameters class. + unshaped_point = scipy.optimize.fmin_cobyla( + func=self._scipy_decorator(self.objective_function.compute_objective_function, **kwargs), + x0=self.objective_function.current_point.flatten(), + cons=self.domain.get_constraint_list(), + rhobeg=self.optimization_parameters.rhobeg, + rhoend=self.optimization_parameters.rhoend, + maxfun=self.optimization_parameters.maxfun, + catol=self.optimization_parameters.catol, + disp=0, # Suppresses output from the routine. + ) + if self._num_points == 1: + shaped_point = unshaped_point + else: + shaped_point = unshaped_point.reshape(self._num_points, self.domain.dim) + self.objective_function.current_point = shaped_point diff --git a/moe/optimal_learning/python/repeated_domain.py b/moe/optimal_learning/python/repeated_domain.py index 2c98e401..1e1eca28 100644 --- a/moe/optimal_learning/python/repeated_domain.py +++ b/moe/optimal_learning/python/repeated_domain.py @@ -79,6 +79,13 @@ def get_bounding_box(self): """Return a list of ClosedIntervals representing a bounding box for this domain.""" return self._domain.get_bounding_box() + def get_constraint_list(self, start_index=0): + """Return a list of lambda functions expressing the domain bounds as linear constraints. Used by COBYLA.""" + constraints = [] + for i in xrange(self.num_repeats): + constraints.extend(self._domain.get_constraint_list(start_index=self._domain.dim * i)) + return constraints + def generate_random_point_in_domain(self, random_source=None): """Generate ``point`` uniformly at random such that ``self.check_point_inside(point)`` is True. diff --git a/moe/tests/optimal_learning/python/python_version/optimization_test.py b/moe/tests/optimal_learning/python/python_version/optimization_test.py index ca0f72f8..5c5c75f3 100644 --- a/moe/tests/optimal_learning/python/python_version/optimization_test.py +++ b/moe/tests/optimal_learning/python/python_version/optimization_test.py @@ -7,7 +7,7 @@ from moe.optimal_learning.python.geometry_utils import ClosedInterval from moe.optimal_learning.python.interfaces.optimization_interface import OptimizableInterface from moe.optimal_learning.python.python_version.domain import TensorProductDomain -from moe.optimal_learning.python.python_version.optimization import multistart_optimize, LBFGSBParameters, GradientDescentParameters, NullOptimizer, GradientDescentOptimizer, MultistartOptimizer, LBFGSBOptimizer +from moe.optimal_learning.python.python_version.optimization import multistart_optimize, LBFGSBParameters, GradientDescentParameters, NullOptimizer, GradientDescentOptimizer, MultistartOptimizer, LBFGSBOptimizer, ConstrainedDFOOptimizer, ConstrainedDFOParameters from moe.tests.optimal_learning.python.optimal_learning_test_case import OptimalLearningTestCase @@ -199,6 +199,17 @@ def base_setup(self): epsilon, ) + maxfun = 1000 + rhobeg = 1.0 + rhoend = 1.0e-13 + catol = 2.0e-13 + self.COBYLA_parameters = ConstrainedDFOParameters( + rhobeg, + rhoend, + maxfun, + catol, + ) + def test_gradient_descent_optimizer(self): """Check that gradient descent can find the optimum of the quadratic test objective.""" # Check the claimed optima is an optima @@ -382,7 +393,7 @@ def test_bfgs_optimizer(self): gradient = self.polynomial.compute_grad_objective_function() self.assert_vector_within_relative(gradient, numpy.zeros(self.polynomial.dim), 0.0) - # Verify that gradient descent does not move from the optima if we start it there. + # Verify that BFGS does not move from the optima if we start it there. bfgs_optimizer = LBFGSBOptimizer(self.domain, self.polynomial, self.BFGS_parameters) bfgs_optimizer.optimize() output = bfgs_optimizer.objective_function.current_point @@ -406,8 +417,8 @@ def test_bfgs_optimizer(self): self.assert_vector_within_relative(gradient, numpy.zeros(self.polynomial.dim), tolerance) def test_multistarted_bfgs_optimizer(self): - """Check that multistarted GD can find the optimum in a 'very' large domain.""" - # Set a large domain: a single GD run is unlikely to reach the optimum + """Check that multistarted BFGS can find the optimum in a 'very' large domain.""" + # Set a large domain: a single BFGS run is unlikely to reach the optimum domain_bounds = [ClosedInterval(-10.0, 10.0)] * self.dim domain = TensorProductDomain(domain_bounds) @@ -427,3 +438,57 @@ def test_multistarted_bfgs_optimizer(self): # Verify derivative gradient = self.polynomial.compute_grad_objective_function() self.assert_vector_within_relative(gradient, numpy.zeros(self.polynomial.dim), tolerance) + + def test_cobyla_optimizer(self): + """Check that COBYLA can find the optimum of the quadratic test objective.""" + # Check the claimed optima is an optima + optimum_point = self.polynomial.optimum_point + self.polynomial.current_point = optimum_point + gradient = self.polynomial.compute_grad_objective_function() + self.assert_vector_within_relative(gradient, numpy.zeros(self.polynomial.dim), 0.0) + + # Verify that COBYLA does not move from the optima if we start it there. + tolerance = 2.0e-13 + constrained_optimizer = ConstrainedDFOOptimizer(self.domain, self.polynomial, self.COBYLA_parameters) + constrained_optimizer.optimize() + output = constrained_optimizer.objective_function.current_point + self.assert_vector_within_relative(output, optimum_point, tolerance) + + # Start at a wrong point and check optimization + initial_guess = numpy.full(self.polynomial.dim, 0.2) + constrained_optimizer.objective_function.current_point = initial_guess + constrained_optimizer.optimize() + output = constrained_optimizer.objective_function.current_point + # Verify coordinates + self.assert_vector_within_relative(output, optimum_point, tolerance) + + # Verify function value + value = self.polynomial.compute_objective_function() + self.assert_scalar_within_relative(value, self.polynomial.optimum_value, tolerance) + + # Verify derivative + gradient = self.polynomial.compute_grad_objective_function() + self.assert_vector_within_relative(gradient, numpy.zeros(self.polynomial.dim), tolerance) + + def test_multistarted_cobyla_optimizer(self): + """Check that multistarted COBYLA can find the optimum in a 'very' large domain.""" + # Set a large domain: a single COBYLA run is unlikely to reach the optimum + domain_bounds = [ClosedInterval(-10.0, 10.0)] * self.dim + domain = TensorProductDomain(domain_bounds) + + tolerance = 2.0e-10 + num_points = 10 + constrained_optimizer = ConstrainedDFOOptimizer(domain, self.polynomial, self.COBYLA_parameters) + multistart_optimizer = MultistartOptimizer(constrained_optimizer, num_points) + + output, _ = multistart_optimizer.optimize() + # Verify coordinates + self.assert_vector_within_relative(output, self.polynomial.optimum_point, tolerance) + + # Verify function value + value = self.polynomial.compute_objective_function() + self.assert_scalar_within_relative(value, self.polynomial.optimum_value, tolerance) + + # Verify derivative + gradient = self.polynomial.compute_grad_objective_function() + self.assert_vector_within_relative(gradient, numpy.zeros(self.polynomial.dim), tolerance) From f4020a90595f113010a4cea50457ffc6c0559cfa Mon Sep 17 00:00:00 2001 From: Deniz Oktay Date: Thu, 21 Aug 2014 17:28:00 -0700 Subject: [PATCH 2/9] addressed views and changed around test file --- moe/optimal_learning/python/constant.py | 1 - .../python/cpp_wrappers/domain.py | 8 + .../python/interfaces/domain_interface.py | 5 + .../python/python_version/domain.py | 2 +- .../python_version/optimization_test.py | 163 +++++------------- 5 files changed, 55 insertions(+), 124 deletions(-) diff --git a/moe/optimal_learning/python/constant.py b/moe/optimal_learning/python/constant.py index 71d8c518..a40192f9 100644 --- a/moe/optimal_learning/python/constant.py +++ b/moe/optimal_learning/python/constant.py @@ -37,7 +37,6 @@ # Domain constants TENSOR_PRODUCT_DOMAIN_TYPE = 'tensor_product' SIMPLEX_INTERSECT_TENSOR_PRODUCT_DOMAIN_TYPE = 'simplex_intersect_tensor_product' -LINEAR_CONSTRAINT_DOMAIN_TYPE = 'linear_constraints' #: Domain types supported by :mod:`moe` DOMAIN_TYPES = [ diff --git a/moe/optimal_learning/python/cpp_wrappers/domain.py b/moe/optimal_learning/python/cpp_wrappers/domain.py index cc9bf195..ba69446d 100644 --- a/moe/optimal_learning/python/cpp_wrappers/domain.py +++ b/moe/optimal_learning/python/cpp_wrappers/domain.py @@ -75,6 +75,14 @@ def get_bounding_box(self): """Return a list of ClosedIntervals representing a bounding box for this domain.""" return copy.copy(self._domain_bounds) + def get_constraint_list(self, start_index=0): + """Return a list of lambda functions expressing the domain bounds as linear constraints. Used by COBYLA.""" + constraints = [] + for i, interval in enumerate(self._domain_bounds): + constraints.append((lambda x: x[i + start_index] - interval.min)) + constraints.append((lambda x: interval.max - x[i + start_index])) + return constraints + def generate_random_point_in_domain(self, random_source=None): """Generate ``point`` uniformly at random such that ``self.check_point_inside(point)`` is True. diff --git a/moe/optimal_learning/python/interfaces/domain_interface.py b/moe/optimal_learning/python/interfaces/domain_interface.py index 9496917b..6866cf4f 100644 --- a/moe/optimal_learning/python/interfaces/domain_interface.py +++ b/moe/optimal_learning/python/interfaces/domain_interface.py @@ -31,6 +31,11 @@ def get_bounding_box(self): """Return a list of ClosedIntervals representing a bounding box for this domain.""" pass + @abstractmethod + def get_constraint_list(self, start_index=0): + """Return a list of lambda functions expressing the domain bounds as linear constraints. Used by COBYLA.""" + pass + @abstractmethod def generate_random_point_in_domain(self, random_source=None): """Generate ``point`` uniformly at random such that ``self.check_point_inside(point)`` is True. diff --git a/moe/optimal_learning/python/python_version/domain.py b/moe/optimal_learning/python/python_version/domain.py index 04058a05..da4e10e5 100644 --- a/moe/optimal_learning/python/python_version/domain.py +++ b/moe/optimal_learning/python/python_version/domain.py @@ -17,7 +17,7 @@ import numpy -from moe.optimal_learning.python.constant import TENSOR_PRODUCT_DOMAIN_TYPE, LINEAR_CONSTRAINT_DOMAIN_TYPE +from moe.optimal_learning.python.constant import TENSOR_PRODUCT_DOMAIN_TYPE from moe.optimal_learning.python.geometry_utils import generate_grid_points, generate_latin_hypercube_points from moe.optimal_learning.python.interfaces.domain_interface import DomainInterface diff --git a/moe/tests/optimal_learning/python/python_version/optimization_test.py b/moe/tests/optimal_learning/python/python_version/optimization_test.py index 5c5c75f3..7760346a 100644 --- a/moe/tests/optimal_learning/python/python_version/optimization_test.py +++ b/moe/tests/optimal_learning/python/python_version/optimization_test.py @@ -163,6 +163,9 @@ def base_setup(self): domain_bounds = [ClosedInterval(-1.0, 1.0)] * self.dim self.domain = TensorProductDomain(domain_bounds) + large_domain_bounds = [ClosedInterval(-1.0, 1.0)] * self.dim + self.large_domain = TensorProductDomain(large_domain_bounds) + maxima_point = numpy.full(self.dim, 0.5) current_point = numpy.zeros(self.dim) self.polynomial = QuadraticFunction(maxima_point, current_point) @@ -210,37 +213,6 @@ def base_setup(self): catol, ) - def test_gradient_descent_optimizer(self): - """Check that gradient descent can find the optimum of the quadratic test objective.""" - # Check the claimed optima is an optima - optimum_point = self.polynomial.optimum_point - self.polynomial.current_point = optimum_point - gradient = self.polynomial.compute_grad_objective_function() - self.assert_vector_within_relative(gradient, numpy.zeros(self.polynomial.dim), 0.0) - - # Verify that gradient descent does not move from the optima if we start it there. - gradient_descent_optimizer = GradientDescentOptimizer(self.domain, self.polynomial, self.gd_parameters) - gradient_descent_optimizer.optimize() - output = gradient_descent_optimizer.objective_function.current_point - self.assert_vector_within_relative(output, optimum_point, 0.0) - - # Start at a wrong point and check optimization - tolerance = 2.0e-13 - initial_guess = numpy.full(self.polynomial.dim, 0.2) - gradient_descent_optimizer.objective_function.current_point = initial_guess - gradient_descent_optimizer.optimize() - output = gradient_descent_optimizer.objective_function.current_point - # Verify coordinates - self.assert_vector_within_relative(output, optimum_point, tolerance) - - # Verify function value - value = self.polynomial.compute_objective_function() - self.assert_scalar_within_relative(value, self.polynomial.optimum_value, tolerance) - - # Verify derivative - gradient = self.polynomial.compute_grad_objective_function() - self.assert_vector_within_relative(gradient, numpy.zeros(self.polynomial.dim), tolerance) - def test_get_averaging_range(self): """Test the method used to produce what interval to average over in Polyak-Ruppert averaging.""" num_steps_total = 250 @@ -362,49 +334,25 @@ def test_multistarted_gradient_descent_optimizer_crippled_start(self): for value in (test_best_point - self.polynomial.optimum_point): T.assert_equal(value, 0.0) - def test_multistarted_gradient_descent_optimizer(self): - """Check that multistarted GD can find the optimum in a 'very' large domain.""" - # Set a large domain: a single GD run is unlikely to reach the optimum - domain_bounds = [ClosedInterval(-10.0, 10.0)] * self.dim - domain = TensorProductDomain(domain_bounds) - - tolerance = 2.0e-10 - num_points = 10 - gradient_descent_optimizer = GradientDescentOptimizer(domain, self.polynomial, self.gd_parameters) - multistart_optimizer = MultistartOptimizer(gradient_descent_optimizer, num_points) - - output, _ = multistart_optimizer.optimize() - # Verify coordinates - self.assert_vector_within_relative(output, self.polynomial.optimum_point, tolerance) - - # Verify function value - value = self.polynomial.compute_objective_function() - self.assert_scalar_within_relative(value, self.polynomial.optimum_value, tolerance) - - # Verify derivative - gradient = self.polynomial.compute_grad_objective_function() - self.assert_vector_within_relative(gradient, numpy.zeros(self.polynomial.dim), tolerance) - - def test_bfgs_optimizer(self): - """Check that BFGS can find the optimum of the quadratic test objective.""" + def optimizer_test(self, optimizer): + """Check that the optimizer can find the optimum of the quadratic test objective.""" # Check the claimed optima is an optima optimum_point = self.polynomial.optimum_point self.polynomial.current_point = optimum_point gradient = self.polynomial.compute_grad_objective_function() self.assert_vector_within_relative(gradient, numpy.zeros(self.polynomial.dim), 0.0) - # Verify that BFGS does not move from the optima if we start it there. - bfgs_optimizer = LBFGSBOptimizer(self.domain, self.polynomial, self.BFGS_parameters) - bfgs_optimizer.optimize() - output = bfgs_optimizer.objective_function.current_point - self.assert_vector_within_relative(output, optimum_point, 0.0) + # Verify that the optimizer does not move from the optima if we start it there. + tolerance = 2.0e-13 + optimizer.optimize() + output = optimizer.objective_function.current_point + self.assert_vector_within_relative(output, optimum_point, tolerance) # Start at a wrong point and check optimization - tolerance = 2.0e-13 initial_guess = numpy.full(self.polynomial.dim, 0.2) - bfgs_optimizer.objective_function.current_point = initial_guess - bfgs_optimizer.optimize() - output = bfgs_optimizer.objective_function.current_point + optimizer.objective_function.current_point = initial_guess + optimizer.optimize() + output = optimizer.objective_function.current_point # Verify coordinates self.assert_vector_within_relative(output, optimum_point, tolerance) @@ -416,16 +364,11 @@ def test_bfgs_optimizer(self): gradient = self.polynomial.compute_grad_objective_function() self.assert_vector_within_relative(gradient, numpy.zeros(self.polynomial.dim), tolerance) - def test_multistarted_bfgs_optimizer(self): - """Check that multistarted BFGS can find the optimum in a 'very' large domain.""" - # Set a large domain: a single BFGS run is unlikely to reach the optimum - domain_bounds = [ClosedInterval(-10.0, 10.0)] * self.dim - domain = TensorProductDomain(domain_bounds) - + def multistarted_optimizer_test(self, optimizer): + """Check that the multistarted optimizer can find the optimum in a 'very' large domain.""" tolerance = 2.0e-10 num_points = 10 - bfgs_optimizer = LBFGSBOptimizer(domain, self.polynomial, self.BFGS_parameters) - multistart_optimizer = MultistartOptimizer(bfgs_optimizer, num_points) + multistart_optimizer = MultistartOptimizer(optimizer, num_points) output, _ = multistart_optimizer.optimize() # Verify coordinates @@ -440,55 +383,31 @@ def test_multistarted_bfgs_optimizer(self): self.assert_vector_within_relative(gradient, numpy.zeros(self.polynomial.dim), tolerance) def test_cobyla_optimizer(self): - """Check that COBYLA can find the optimum of the quadratic test objective.""" - # Check the claimed optima is an optima - optimum_point = self.polynomial.optimum_point - self.polynomial.current_point = optimum_point - gradient = self.polynomial.compute_grad_objective_function() - self.assert_vector_within_relative(gradient, numpy.zeros(self.polynomial.dim), 0.0) - - # Verify that COBYLA does not move from the optima if we start it there. - tolerance = 2.0e-13 + """Test if COBYLA can optimize a simple objective function.""" constrained_optimizer = ConstrainedDFOOptimizer(self.domain, self.polynomial, self.COBYLA_parameters) - constrained_optimizer.optimize() - output = constrained_optimizer.objective_function.current_point - self.assert_vector_within_relative(output, optimum_point, tolerance) - - # Start at a wrong point and check optimization - initial_guess = numpy.full(self.polynomial.dim, 0.2) - constrained_optimizer.objective_function.current_point = initial_guess - constrained_optimizer.optimize() - output = constrained_optimizer.objective_function.current_point - # Verify coordinates - self.assert_vector_within_relative(output, optimum_point, tolerance) - - # Verify function value - value = self.polynomial.compute_objective_function() - self.assert_scalar_within_relative(value, self.polynomial.optimum_value, tolerance) - - # Verify derivative - gradient = self.polynomial.compute_grad_objective_function() - self.assert_vector_within_relative(gradient, numpy.zeros(self.polynomial.dim), tolerance) - - def test_multistarted_cobyla_optimizer(self): - """Check that multistarted COBYLA can find the optimum in a 'very' large domain.""" - # Set a large domain: a single COBYLA run is unlikely to reach the optimum - domain_bounds = [ClosedInterval(-10.0, 10.0)] * self.dim - domain = TensorProductDomain(domain_bounds) - - tolerance = 2.0e-10 - num_points = 10 - constrained_optimizer = ConstrainedDFOOptimizer(domain, self.polynomial, self.COBYLA_parameters) - multistart_optimizer = MultistartOptimizer(constrained_optimizer, num_points) - - output, _ = multistart_optimizer.optimize() - # Verify coordinates - self.assert_vector_within_relative(output, self.polynomial.optimum_point, tolerance) + self.optimizer_test(constrained_optimizer) - # Verify function value - value = self.polynomial.compute_objective_function() - self.assert_scalar_within_relative(value, self.polynomial.optimum_value, tolerance) + def test_bfgs_optimizer(self): + """Test if BFGS can optimize a simple objective function.""" + bfgs_optimizer = LBFGSBOptimizer(self.domain, self.polynomial, self.BFGS_parameters) + self.optimizer_test(bfgs_optimizer) - # Verify derivative - gradient = self.polynomial.compute_grad_objective_function() - self.assert_vector_within_relative(gradient, numpy.zeros(self.polynomial.dim), tolerance) + def test_gradient_descent_optimizer(self): + """Test if Gradient Descent can optimize a simple objective function.""" + gradient_descent_optimizer = GradientDescentOptimizer(self.domain, self.polynomial, self.gd_parameters) + self.optimizer_test(gradient_descent_optimizer) + + def test_cobyla_multistarted_optimizer(self): + """Test if COBYLA can optimize a "hard" objective function with multistarts.""" + constrained_optimizer = ConstrainedDFOOptimizer(self.large_domain, self.polynomial, self.COBYLA_parameters) + self.multistarted_optimizer_test(constrained_optimizer) + + def test_bfgs_multistarted_optimizer(self): + """Test if BFGS can optimize a "hard" objective function with multistarts.""" + bfgs_optimizer = LBFGSBOptimizer(self.large_domain, self.polynomial, self.BFGS_parameters) + self.multistarted_optimizer_test(bfgs_optimizer) + + def test_gradient_descent_multistarted_optimizer(self): + """Test if Gradient Descent can optimize a "hard" objective function with multistarts.""" + gradient_descent_optimizer = GradientDescentOptimizer(self.large_domain, self.polynomial, self.gd_parameters) + self.multistarted_optimizer_test(gradient_descent_optimizer) From 3fc8bb7f79444a25b82117341853cd03436aea93 Mon Sep 17 00:00:00 2001 From: Deniz Oktay Date: Thu, 21 Aug 2014 17:39:11 -0700 Subject: [PATCH 3/9] added comments for start_point --- moe/optimal_learning/python/python_version/domain.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/moe/optimal_learning/python/python_version/domain.py b/moe/optimal_learning/python/python_version/domain.py index da4e10e5..54d0c7e0 100644 --- a/moe/optimal_learning/python/python_version/domain.py +++ b/moe/optimal_learning/python/python_version/domain.py @@ -86,7 +86,12 @@ def get_bounding_box(self): return copy.copy(self._domain_bounds) def get_constraint_list(self, start_index=0): - """Return a list of lambda functions expressing the domain bounds as linear constraints. Used by COBYLA.""" + """Return a list of lambda functions expressing the domain bounds as linear constraints. Used by COBYLA. + + Since COBYLA in scipy only optimizes arrays, we flatten out our points while doing multipoint EI optimization. + But in order for the constraints to access the correct index, the RepeatedDomain class has to signal which index + the TensorProductDomain should start from, using the start_index optional parameter. + """ constraints = [] for i, interval in enumerate(self._domain_bounds): constraints.append((lambda x: x[i + start_index] - interval.min)) From c38a678d9465426e596720ab8f0028b45058cce7 Mon Sep 17 00:00:00 2001 From: Deniz Oktay Date: Fri, 22 Aug 2014 14:06:17 -0700 Subject: [PATCH 4/9] addressed reviews --- .../python/python_version/domain.py | 10 ++++++-- .../python/python_version/optimization.py | 24 ++++++++++++------- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/moe/optimal_learning/python/python_version/domain.py b/moe/optimal_learning/python/python_version/domain.py index 54d0c7e0..b624a06a 100644 --- a/moe/optimal_learning/python/python_version/domain.py +++ b/moe/optimal_learning/python/python_version/domain.py @@ -87,10 +87,16 @@ def get_bounding_box(self): def get_constraint_list(self, start_index=0): """Return a list of lambda functions expressing the domain bounds as linear constraints. Used by COBYLA. - - Since COBYLA in scipy only optimizes arrays, we flatten out our points while doing multipoint EI optimization. + + Since COBYLA in scipy only optimizes arrays, we flatten out our points while doing multipoint EI optimization. But in order for the constraints to access the correct index, the RepeatedDomain class has to signal which index the TensorProductDomain should start from, using the start_index optional parameter. + + :param start_index: the dimension this tensor product domain should start indexing from + :type start_index: integer (>= 0) + :return: a list of lambda functions corresponding to constraints + :rtype: array of lambda functions with shape (dim * 2) + """ constraints = [] for i, interval in enumerate(self._domain_bounds): diff --git a/moe/optimal_learning/python/python_version/optimization.py b/moe/optimal_learning/python/python_version/optimization.py index f1a9b9e1..de8dc313 100644 --- a/moe/optimal_learning/python/python_version/optimization.py +++ b/moe/optimal_learning/python/python_version/optimization.py @@ -337,7 +337,8 @@ class ConstrainedDFOParameters(_BaseConstrainedDFOParameters): r"""Container to hold parameters that specify the behavior of COBYLA. - Suggested values come from scipy documentation for scipy.optimize.fmin_cobyla. + Suggested values come from scipy documentation for scipy.optimize.fmin_cobyla: + http://docs.scipy.org/doc/scipy-0.13.0/reference/generated/scipy.optimize.fmin_cobyla.html :ivar rhobeg: (*float64 > 0.0*) reasonable initial changes to the variables (suggest: 1.0) :ivar rhoend: (*float64 > 0.0*) final accuracy in the optimization (not precisely guaranteed), which is a lower bound on the size of the trust region (suggest: 1.0e-4) @@ -624,15 +625,16 @@ def __init__(self, domain, optimizable, optimization_parameters, num_random_samp def _scipy_decorator(self, func, **kwargs): """Wrapper function for expected improvement calculation to feed into BFGS. - func should be of the form compute_* in interfaces.optimization_interface.OptimizableInterface. + func should be of the form `compute_*` in :class:`moe.optimal_learning.python.interfaces.optimization_interface.OptimizableInterface`. """ def decorated(point): """Decorator for compute_* functions in interfaces.optimization_interface.OptimizableInterface. - Converts the point to proper format and sets the current point before calling the compute function. + Converts the point to proper format (array with dim (self._num_points, self.domain.dim) instead of flat array) + and sets the current point before calling the compute function. :param point: the point on which to do the calculation - :type point: array of float64 with shape (self._num_points * self.domain.dim) + :type point: array of float64 with shape (self._num_points * self.domain.dim, ) """ shaped_point = point.reshape(self._num_points, self.domain.dim) self.objective_function.current_point = shaped_point @@ -677,7 +679,7 @@ class ConstrainedDFOOptimizer(OptimizerInterface): r"""Optimizes an objective function over the specified contraints with the COBYLA method. - .. Note:: See optimize() docstring for more details. + .. Note:: See :func:`~moe.optimal_learning.python.python_version.optimization.ConstrainedDFOOptimizer.optimize()` docstring for more details. """ @@ -696,18 +698,20 @@ def __init__(self, domain, optimizable, optimization_parameters, num_random_samp self.objective_function = optimizable self.optimization_parameters = optimization_parameters self._num_points = 1 + # Check if this is a repeated domain, and if so set points equal to number of repeats. if hasattr(self.domain, 'num_repeats'): self._num_points = self.domain.num_repeats def _scipy_decorator(self, func, **kwargs): """Wrapper function for expected improvement calculation to feed into COBYLA. - func should be of the form compute_* in interfaces.optimization_interface.OptimizableInterface. + func should be of the form compute_* in :class:`moe.optimal_learning.python.interfaces.optimization_interface.OptimizableInterface`. """ def decorated(point): """Decorator for compute_* functions in interfaces.optimization_interface.OptimizableInterface. - Converts the point to proper format and sets the current point before calling the compute function. + Converts the point to proper format (array with dim (self._num_points, self.domain.dim) instead of flat array) + and sets the current point before calling the compute function. :param point: the point on which to do the calculation :type point: array of float64 with shape (self._num_points * self.domain.dim) @@ -725,9 +729,13 @@ def decorated(point): def optimize(self, **kwargs): """Perform a COBYLA optimization given the parameters in optimization_parameters. + For more information, visit the scipy docs page and the original paper by Powell: + http://docs.scipy.org/doc/scipy-0.13.0/reference/generated/scipy.optimize.fmin_cobyla.html + http://www.damtp.cam.ac.uk/user/na/NA_papers/NA2007_03.pdf + objective_function.current_point will be set to the optimal point found. """ - # Parameters defined above in LBFGSBParameters class. + # Parameters defined above in :class:`~moe.optimal_learning.python.python_version.optimization.LBFGSBParameters` class. unshaped_point = scipy.optimize.fmin_cobyla( func=self._scipy_decorator(self.objective_function.compute_objective_function, **kwargs), x0=self.objective_function.current_point.flatten(), From 07c214cbb1916b6f3b7d4bbb795b910da601b501 Mon Sep 17 00:00:00 2001 From: Deniz Oktay Date: Fri, 22 Aug 2014 15:33:54 -0700 Subject: [PATCH 5/9] made better scipy wrapper --- .../python/python_version/optimization.py | 158 +++++++++--------- .../python_version/optimization_test.py | 5 +- 2 files changed, 81 insertions(+), 82 deletions(-) diff --git a/moe/optimal_learning/python/python_version/optimization.py b/moe/optimal_learning/python/python_version/optimization.py index de8dc313..5fafb8dc 100644 --- a/moe/optimal_learning/python/python_version/optimization.py +++ b/moe/optimal_learning/python/python_version/optimization.py @@ -173,6 +173,8 @@ fail, we commonly fall back to 'dumb' search. """ +from abc import abstractmethod + import collections import numpy @@ -584,35 +586,23 @@ def optimize(self, random_starts=None, **kwargs): return best_point, function_value_list -class LBFGSBOptimizer(OptimizerInterface): +class _ScipyOptimizerWrapper(OptimizerInterface): - r"""Optimizes an objective function over the specified domain with the L-BFGS-B method. + """Wrapper class to construct an optimizer from scipy optimization methods. - The BFGS (Broyden-Fletcher-Goldfarb-Shanno) algorithm is a quasi-Newton algorithm for optimization. It can - be used for DFO (Derivative-Free Optimization) when the gradient is not available, such as is the case for - the analytic qEI algorithm. - - L-BFGS is a memory efficient version of BFGS, and BFGS-B is a variant that handles simple box constraints. - We use L-BFGS-B, which is a combination of the two, and is often the optimization algorithm of choice for - these types of problems. - - For more information: - http://en.wikipedia.org/wiki/Limited-memory_BFGS - http://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.fmin_l_bfgs_b.html - - .. Note:: See optimize() docstring for more details. + Requires the implementation of the :func:`~moe.optimal_learning.python.python_interface.optimization._ScipyOptimizerWrapper.get_scipy_optimizer_unshaped_point` method. """ - def __init__(self, domain, optimizable, optimization_parameters, num_random_samples=None): - """Construct a LBFGSBOptimizer. + def __init__(self, domain, optimizable, optimization_parameters): + """Construct the optimizer. :param domain: the domain that this optimizer operates over - :type domain: interfaces.domain_interface.DomainInterface subclass. Only supports TensorProductDomain. + :type domain: :class:`~moe.optimal_learning.python.interfaces.domain_interface.DomainInterface` subclass. :param optimizable: object representing the objective function being optimized - :type optimizable: interfaces.optimization_interface.OptimizableInterface subclass + :type optimizable: :class:`~moe.optimal_learning.python.interfaces.optimization_interface.OptimizableInterface` subclass :param optimization_parameters: parameters describing how to perform optimization (tolerances, iterations, etc.) - :type optimization_parameters: python_version.optimization.LBFGSBParameters object + :type optimization_parameters: `python_version.optimization.*Parameters` object """ self.domain = domain @@ -623,7 +613,7 @@ def __init__(self, domain, optimizable, optimization_parameters, num_random_samp self._num_points = self.domain.num_repeats def _scipy_decorator(self, func, **kwargs): - """Wrapper function for expected improvement calculation to feed into BFGS. + """Wrapper function for expected improvement calculation to feed into the optimizer function. func should be of the form `compute_*` in :class:`moe.optimal_learning.python.interfaces.optimization_interface.OptimizableInterface`. """ @@ -647,16 +637,69 @@ def decorated(point): return decorated def optimize(self, **kwargs): - """Perform an L-BFGS-B optimization given the parameters in optimization_parameters. + """Shape the point returned by the get_scipy_optimizer_unshaped_point function. objective_function.current_point will be set to the optimal point found. """ + # Parameters defined above in LBFGSBParameters class. + unshaped_point = self.get_scipy_optimizer_unshaped_point(**kwargs) + + if not isinstance(unshaped_point, numpy.ndarray): + unshaped_point = unshaped_point[0] + if self._num_points == 1: + shaped_point = unshaped_point + else: + shaped_point = unshaped_point.reshape(self._num_points, self.domain.dim) + self.objective_function.current_point = shaped_point + + @abstractmethod + def get_scipy_optimizer_unshaped_point(self, **kwargs): + """Should return an unshaped point corresponding to the output of the optimizer function from scipy. + + See L-BFGS-B or COBYLA function for examples. + """ + pass + + +class LBFGSBOptimizer(_ScipyOptimizerWrapper): + + r"""Optimizes an objective function over the specified domain with the L-BFGS-B method. + + The BFGS (Broyden-Fletcher-Goldfarb-Shanno) algorithm is a quasi-Newton algorithm for optimization. It can + be used for DFO (Derivative-Free Optimization) when the gradient is not available, such as is the case for + the analytic qEI algorithm. + + L-BFGS is a memory efficient version of BFGS, and BFGS-B is a variant that handles simple box constraints. + We use L-BFGS-B, which is a combination of the two, and is often the optimization algorithm of choice for + these types of problems. + + For more information, visit the scipy docs and the wikipedia page on BFGS: + http://en.wikipedia.org/wiki/Limited-memory_BFGS + http://docs.scipy.org/doc/scipy/reference/generated/scipy.optimize.fmin_l_bfgs_b.html + + """ + + def __init__(self, domain, optimizable, optimization_parameters, num_random_samples=None): + """Construct a LBFGSBOptimizer. + + :param domain: the domain that this optimizer operates over + :type domain: :class:`~moe.optimal_learning.python.interfaces.domain_interface.DomainInterface` subclass. Only supports TensorProductDomain. + :param optimizable: object representing the objective function being optimized + :type optimizable: :class:`~moe.optimal_learning.python.interfaces.optimization_interface.OptimizableInterface` subclass + :param optimization_parameters: parameters describing how to perform optimization (tolerances, iterations, etc.) + :type optimization_parameters: :class:`~moe.optimal_learning.python.python_version.optimization.LBFGSBParameters` object + + """ + super(LBFGSBOptimizer, self).__init__(domain, optimizable, optimization_parameters) + + def get_scipy_optimizer_unshaped_point(self, **kwargs): + """Perform an L-BFGS-B optimization given the parameters in optimization_parameters.""" domain_bounding_box = self.domain.get_bounding_box() domain_list = [(interval.min, interval.max) for interval in domain_bounding_box] domain_numpy = numpy.array(domain_list * self._num_points) # Parameters defined above in LBFGSBParameters class. - unshaped_point = scipy.optimize.fmin_l_bfgs_b( + return scipy.optimize.fmin_l_bfgs_b( func=self._scipy_decorator(self.objective_function.compute_objective_function, **kwargs), x0=self.objective_function.current_point.flatten(), bounds=domain_numpy, @@ -668,75 +711,35 @@ def optimize(self, **kwargs): pgtol=self.optimization_parameters.pgtol, epsilon=self.optimization_parameters.epsilon, )[0] - if self._num_points == 1: - shaped_point = unshaped_point - else: - shaped_point = unshaped_point.reshape(self._num_points, self.domain.dim) - self.objective_function.current_point = shaped_point -class ConstrainedDFOOptimizer(OptimizerInterface): +class ConstrainedDFOOptimizer(_ScipyOptimizerWrapper): r"""Optimizes an objective function over the specified contraints with the COBYLA method. - .. Note:: See :func:`~moe.optimal_learning.python.python_version.optimization.ConstrainedDFOOptimizer.optimize()` docstring for more details. + For more information, visit the scipy docs page and the original paper by Powell: + http://docs.scipy.org/doc/scipy-0.13.0/reference/generated/scipy.optimize.fmin_cobyla.html + http://www.damtp.cam.ac.uk/user/na/NA_papers/NA2007_03.pdf """ - def __init__(self, domain, optimizable, optimization_parameters, num_random_samples=None): + def __init__(self, domain, optimizable, optimization_parameters): """Construct a ConstrainedDFOOptimizer. :param domain: the domain that this optimizer operates over - :type domain: interfaces.domain_interface.DomainInterface subclass. Only supports TensorProductDomain for now. + :type domain: :class:`~moe.optimal_learning.python.interfaces.domain_interface.DomainInterface` subclass. Only supports TensorProductDomain for now. :param optimizable: object representing the objective function being optimized - :type optimizable: interfaces.optimization_interface.OptimizableInterface subclass + :type optimizable: :class:`~moe.optimal_learning.python.interfaces.optimization_interface.OptimizableInterface` subclass :param optimization_parameters: parameters describing how to perform optimization (tolerances, iterations, etc.) - :type optimization_parameters: python_version.optimization.ConstrainedDFOParameters object + :type optimization_parameters: :class:`~moe.optimal_learning.python.python_version.optimization.ConstrainedDFOParameters` object """ - self.domain = domain - self.objective_function = optimizable - self.optimization_parameters = optimization_parameters - self._num_points = 1 - # Check if this is a repeated domain, and if so set points equal to number of repeats. - if hasattr(self.domain, 'num_repeats'): - self._num_points = self.domain.num_repeats - - def _scipy_decorator(self, func, **kwargs): - """Wrapper function for expected improvement calculation to feed into COBYLA. - - func should be of the form compute_* in :class:`moe.optimal_learning.python.interfaces.optimization_interface.OptimizableInterface`. - """ - def decorated(point): - """Decorator for compute_* functions in interfaces.optimization_interface.OptimizableInterface. + super(ConstrainedDFOOptimizer, self).__init__(domain, optimizable, optimization_parameters) - Converts the point to proper format (array with dim (self._num_points, self.domain.dim) instead of flat array) - and sets the current point before calling the compute function. - - :param point: the point on which to do the calculation - :type point: array of float64 with shape (self._num_points * self.domain.dim) - """ - shaped_point = point.reshape(self._num_points, self.domain.dim) - self.objective_function.current_point = shaped_point - value = -func(**kwargs) - if isinstance(value, (numpy.ndarray)): - return value.flatten() - else: - return value - - return decorated - - def optimize(self, **kwargs): - """Perform a COBYLA optimization given the parameters in optimization_parameters. - - For more information, visit the scipy docs page and the original paper by Powell: - http://docs.scipy.org/doc/scipy-0.13.0/reference/generated/scipy.optimize.fmin_cobyla.html - http://www.damtp.cam.ac.uk/user/na/NA_papers/NA2007_03.pdf - - objective_function.current_point will be set to the optimal point found. - """ + def get_scipy_optimizer_unshaped_point(self, **kwargs): + """Perform a COBYLA optimization given the parameters in optimization_parameters.""" # Parameters defined above in :class:`~moe.optimal_learning.python.python_version.optimization.LBFGSBParameters` class. - unshaped_point = scipy.optimize.fmin_cobyla( + return scipy.optimize.fmin_cobyla( func=self._scipy_decorator(self.objective_function.compute_objective_function, **kwargs), x0=self.objective_function.current_point.flatten(), cons=self.domain.get_constraint_list(), @@ -746,8 +749,3 @@ def optimize(self, **kwargs): catol=self.optimization_parameters.catol, disp=0, # Suppresses output from the routine. ) - if self._num_points == 1: - shaped_point = unshaped_point - else: - shaped_point = unshaped_point.reshape(self._num_points, self.domain.dim) - self.objective_function.current_point = shaped_point diff --git a/moe/tests/optimal_learning/python/python_version/optimization_test.py b/moe/tests/optimal_learning/python/python_version/optimization_test.py index 7760346a..9f3501d3 100644 --- a/moe/tests/optimal_learning/python/python_version/optimization_test.py +++ b/moe/tests/optimal_learning/python/python_version/optimization_test.py @@ -139,11 +139,12 @@ def test_multistarted_null_optimizer(self): self.assert_vector_within_relative(test_values, truth, 0.0) -class GradientDescentOptimizerTest(OptimalLearningTestCase): +class OptimizerTest(OptimalLearningTestCase): - r"""Test Gradient Descent on a simple quadratic objective. + r"""Test the implemented optimizers on a simple quadratic objective. We check GD in an unconstrained setting, a constrained setting, and we test multistarting it. + For the other optimizers we check them in a constrained setting and a multistarted setting. We don't test the stochastic averaging option meaningfully. We check that the optimizer will average the number of steps specified by input. We also check that the simple unconstrained case can also be solved From ea279eee333b2bc6b36f00e7697d65443b398c2d Mon Sep 17 00:00:00 2001 From: Deniz Oktay Date: Fri, 22 Aug 2014 15:39:18 -0700 Subject: [PATCH 6/9] fixed ticks --- moe/optimal_learning/python/python_version/optimization.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/moe/optimal_learning/python/python_version/optimization.py b/moe/optimal_learning/python/python_version/optimization.py index 5fafb8dc..9ab6563d 100644 --- a/moe/optimal_learning/python/python_version/optimization.py +++ b/moe/optimal_learning/python/python_version/optimization.py @@ -602,7 +602,7 @@ def __init__(self, domain, optimizable, optimization_parameters): :param optimizable: object representing the objective function being optimized :type optimizable: :class:`~moe.optimal_learning.python.interfaces.optimization_interface.OptimizableInterface` subclass :param optimization_parameters: parameters describing how to perform optimization (tolerances, iterations, etc.) - :type optimization_parameters: `python_version.optimization.*Parameters` object + :type optimization_parameters: ``python_version.optimization.*Parameters`` object """ self.domain = domain @@ -615,7 +615,7 @@ def __init__(self, domain, optimizable, optimization_parameters): def _scipy_decorator(self, func, **kwargs): """Wrapper function for expected improvement calculation to feed into the optimizer function. - func should be of the form `compute_*` in :class:`moe.optimal_learning.python.interfaces.optimization_interface.OptimizableInterface`. + func should be of the form ``compute_*`` in :class:`moe.optimal_learning.python.interfaces.optimization_interface.OptimizableInterface`. """ def decorated(point): """Decorator for compute_* functions in interfaces.optimization_interface.OptimizableInterface. From a79da85464be3d3dbe7c931a343b714e499a3ee6 Mon Sep 17 00:00:00 2001 From: Deniz Oktay Date: Fri, 22 Aug 2014 15:51:35 -0700 Subject: [PATCH 7/9] updated changelog, fixed some docstrings --- CHANGELOG.md | 1 + moe/optimal_learning/python/cpp_wrappers/domain.py | 8 ++------ .../python/interfaces/domain_interface.py | 2 +- moe/optimal_learning/python/repeated_domain.py | 3 ++- 4 files changed, 6 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9fd13d16..360bebd6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ * Implemented BLA (Bayesian Learning Automaton). (#373) * Connected GPU functions to multistart gradient descent optimizer. (#270) + * Added the COBYLA optimizer to the expected improvement optimization class. (#370) * Changes diff --git a/moe/optimal_learning/python/cpp_wrappers/domain.py b/moe/optimal_learning/python/cpp_wrappers/domain.py index ba69446d..36f97c1d 100644 --- a/moe/optimal_learning/python/cpp_wrappers/domain.py +++ b/moe/optimal_learning/python/cpp_wrappers/domain.py @@ -75,13 +75,9 @@ def get_bounding_box(self): """Return a list of ClosedIntervals representing a bounding box for this domain.""" return copy.copy(self._domain_bounds) - def get_constraint_list(self, start_index=0): + def get_constraint_list(self): """Return a list of lambda functions expressing the domain bounds as linear constraints. Used by COBYLA.""" - constraints = [] - for i, interval in enumerate(self._domain_bounds): - constraints.append((lambda x: x[i + start_index] - interval.min)) - constraints.append((lambda x: interval.max - x[i + start_index])) - return constraints + raise NotImplementedError("Constraints are not yet implemented for C++.") def generate_random_point_in_domain(self, random_source=None): """Generate ``point`` uniformly at random such that ``self.check_point_inside(point)`` is True. diff --git a/moe/optimal_learning/python/interfaces/domain_interface.py b/moe/optimal_learning/python/interfaces/domain_interface.py index 6866cf4f..0f2f4aed 100644 --- a/moe/optimal_learning/python/interfaces/domain_interface.py +++ b/moe/optimal_learning/python/interfaces/domain_interface.py @@ -32,7 +32,7 @@ def get_bounding_box(self): pass @abstractmethod - def get_constraint_list(self, start_index=0): + def get_constraint_list(self): """Return a list of lambda functions expressing the domain bounds as linear constraints. Used by COBYLA.""" pass diff --git a/moe/optimal_learning/python/repeated_domain.py b/moe/optimal_learning/python/repeated_domain.py index 1e1eca28..6fcfff54 100644 --- a/moe/optimal_learning/python/repeated_domain.py +++ b/moe/optimal_learning/python/repeated_domain.py @@ -79,10 +79,11 @@ def get_bounding_box(self): """Return a list of ClosedIntervals representing a bounding box for this domain.""" return self._domain.get_bounding_box() - def get_constraint_list(self, start_index=0): + def get_constraint_list(self): """Return a list of lambda functions expressing the domain bounds as linear constraints. Used by COBYLA.""" constraints = [] for i in xrange(self.num_repeats): + # Using start_index, start each domain at the correct index when flattening out points in COBYLA. constraints.extend(self._domain.get_constraint_list(start_index=self._domain.dim * i)) return constraints From 05bf53cf1930e5b75660670e0ad0c66f8c8e9784 Mon Sep 17 00:00:00 2001 From: Deniz Oktay Date: Fri, 22 Aug 2014 16:32:30 -0700 Subject: [PATCH 8/9] its the final commit of the summer\! --- .../python/python_version/optimization.py | 24 ++++++++++++------- .../python/repeated_domain.py | 7 +++++- 2 files changed, 22 insertions(+), 9 deletions(-) diff --git a/moe/optimal_learning/python/python_version/optimization.py b/moe/optimal_learning/python/python_version/optimization.py index 9ab6563d..4619a93d 100644 --- a/moe/optimal_learning/python/python_version/optimization.py +++ b/moe/optimal_learning/python/python_version/optimization.py @@ -616,6 +616,7 @@ def _scipy_decorator(self, func, **kwargs): """Wrapper function for expected improvement calculation to feed into the optimizer function. func should be of the form ``compute_*`` in :class:`moe.optimal_learning.python.interfaces.optimization_interface.OptimizableInterface`. + """ def decorated(point): """Decorator for compute_* functions in interfaces.optimization_interface.OptimizableInterface. @@ -625,6 +626,7 @@ def decorated(point): :param point: the point on which to do the calculation :type point: array of float64 with shape (self._num_points * self.domain.dim, ) + """ shaped_point = point.reshape(self._num_points, self.domain.dim) self.objective_function.current_point = shaped_point @@ -639,10 +641,11 @@ def decorated(point): def optimize(self, **kwargs): """Shape the point returned by the get_scipy_optimizer_unshaped_point function. + Calls the get_scipy_optimizer_unshaped_point method. objective_function.current_point will be set to the optimal point found. + """ - # Parameters defined above in LBFGSBParameters class. - unshaped_point = self.get_scipy_optimizer_unshaped_point(**kwargs) + unshaped_point = self._get_scipy_optimizer_unshaped_point(**kwargs) if not isinstance(unshaped_point, numpy.ndarray): unshaped_point = unshaped_point[0] @@ -653,10 +656,15 @@ def optimize(self, **kwargs): self.objective_function.current_point = shaped_point @abstractmethod - def get_scipy_optimizer_unshaped_point(self, **kwargs): + def _get_scipy_optimizer_unshaped_point(self, **kwargs): """Should return an unshaped point corresponding to the output of the optimizer function from scipy. - See L-BFGS-B or COBYLA function for examples. + See :func:`~moe.optimal_learning.python.python_version.optimization.LBFGSOptimizer.get_scipy_optimizer_unshaped_point` or + :func:`~moe.optimal_learning.python.python_version.optimization.ConstrainedDFOOptimizer.get_scipy_optimizer_unshaped_point` function for examples. + + :return: The unshaped optimal point from calling the scipy optimization method. + :rtype: array of float64 with shape (self._num_points, self.domain.dim) + """ pass @@ -692,13 +700,13 @@ def __init__(self, domain, optimizable, optimization_parameters, num_random_samp """ super(LBFGSBOptimizer, self).__init__(domain, optimizable, optimization_parameters) - def get_scipy_optimizer_unshaped_point(self, **kwargs): + def _get_scipy_optimizer_unshaped_point(self, **kwargs): """Perform an L-BFGS-B optimization given the parameters in optimization_parameters.""" domain_bounding_box = self.domain.get_bounding_box() domain_list = [(interval.min, interval.max) for interval in domain_bounding_box] domain_numpy = numpy.array(domain_list * self._num_points) - # Parameters defined above in LBFGSBParameters class. + # Parameters defined above in :class:`~moe.optimal_learning.python.python_version.optimization.LBFGSBParameters` class. return scipy.optimize.fmin_l_bfgs_b( func=self._scipy_decorator(self.objective_function.compute_objective_function, **kwargs), x0=self.objective_function.current_point.flatten(), @@ -736,9 +744,9 @@ def __init__(self, domain, optimizable, optimization_parameters): """ super(ConstrainedDFOOptimizer, self).__init__(domain, optimizable, optimization_parameters) - def get_scipy_optimizer_unshaped_point(self, **kwargs): + def _get_scipy_optimizer_unshaped_point(self, **kwargs): """Perform a COBYLA optimization given the parameters in optimization_parameters.""" - # Parameters defined above in :class:`~moe.optimal_learning.python.python_version.optimization.LBFGSBParameters` class. + # Parameters defined above in :class:`~moe.optimal_learning.python.python_version.optimization.ConstrainedDFOParameters` class. return scipy.optimize.fmin_cobyla( func=self._scipy_decorator(self.objective_function.compute_objective_function, **kwargs), x0=self.objective_function.current_point.flatten(), diff --git a/moe/optimal_learning/python/repeated_domain.py b/moe/optimal_learning/python/repeated_domain.py index 6fcfff54..718e1a14 100644 --- a/moe/optimal_learning/python/repeated_domain.py +++ b/moe/optimal_learning/python/repeated_domain.py @@ -80,7 +80,12 @@ def get_bounding_box(self): return self._domain.get_bounding_box() def get_constraint_list(self): - """Return a list of lambda functions expressing the domain bounds as linear constraints. Used by COBYLA.""" + """Return a list of lambda functions expressing the domain bounds as linear constraints. Used by COBYLA. + + :return: a list of lambda functions corresponding to constraints + :rtype: array of lambda functions with shape (dim * 2) + + """ constraints = [] for i in xrange(self.num_repeats): # Using start_index, start each domain at the correct index when flattening out points in COBYLA. From 506af3e1f47b6ad8b0f28ce1dc94173017934e92 Mon Sep 17 00:00:00 2001 From: Eric Liu Date: Fri, 7 Nov 2014 16:31:09 -0800 Subject: [PATCH 9/9] cleaning up cobyla branch --- CHANGELOG.md | 2 +- ...stic_expected_improvement_optimization.hpp | 2 +- .../python/interfaces/domain_interface.py | 9 +- .../python/python_version/domain.py | 9 +- .../python/python_version/gaussian_process.py | 4 +- .../python/python_version/optimization.py | 96 +++++++++++-------- .../python/repeated_domain.py | 7 +- .../python_version/optimization_test.py | 37 +++---- 8 files changed, 95 insertions(+), 71 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1267b8fa..3245259b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ * Added startup message to REST server including tips for OSX users (#400) * Added GPU support to ``cpp_wrappers.expected_improvement.multistart_expected_improvement_optimization``; requires ``max_num_threads == 1`` until future multi-GPU support (#368) + * Added the COBYLA optimizer to the expected improvement optimization class. (#370) * Changes @@ -18,7 +19,6 @@ SHA: ``ab6f959c11a0cacbed6dad618fe6ffed71092116`` * Implemented BLA (Bayesian Learning Automaton). (#373) * Connected GPU functions to multistart gradient descent optimizer. (#270) - * Added the COBYLA optimizer to the expected improvement optimization class. (#370) * Changes diff --git a/moe/optimal_learning/cpp/gpp_heuristic_expected_improvement_optimization.hpp b/moe/optimal_learning/cpp/gpp_heuristic_expected_improvement_optimization.hpp index e7344b00..78655835 100644 --- a/moe/optimal_learning/cpp/gpp_heuristic_expected_improvement_optimization.hpp +++ b/moe/optimal_learning/cpp/gpp_heuristic_expected_improvement_optimization.hpp @@ -152,7 +152,7 @@ class ObjectiveEstimationPolicyInterface { /*!\rst The "Constant Liar" objective function estimation policy is the simplest: it always returns the same value - (Ginsbourger 2008). We call this the "lie. This object also allows users to associate a noise variance to + (Ginsbourger 2008). We call this the "lie". This object also allows users to associate a noise variance to the lie value. In Ginsbourger's work, the most common lie values have been the min and max of all previously observed objective diff --git a/moe/optimal_learning/python/interfaces/domain_interface.py b/moe/optimal_learning/python/interfaces/domain_interface.py index 0f2f4aed..b06ed693 100644 --- a/moe/optimal_learning/python/interfaces/domain_interface.py +++ b/moe/optimal_learning/python/interfaces/domain_interface.py @@ -33,7 +33,12 @@ def get_bounding_box(self): @abstractmethod def get_constraint_list(self): - """Return a list of lambda functions expressing the domain bounds as linear constraints. Used by COBYLA.""" + """Return a list of lambda functions expressing the domain bounds as linear constraints. Used by COBYLA. + + :return: a list of lambda functions corresponding to constraints + :rtype: array of lambda functions with shape (dim * 2) + + """ pass @abstractmethod @@ -64,7 +69,7 @@ def generate_uniform_random_points_in_domain(self, num_points, random_source): fewer than ``num_points`` results. :param num_points: max number of points to generate - :type num_points: integer >= 0 + :type num_points: int >= 0 :param random_source: :type random_source: callable yielding uniform random numbers in [0,1] :return: uniform random sampling of points from the domain; may be fewer than ``num_points``! diff --git a/moe/optimal_learning/python/python_version/domain.py b/moe/optimal_learning/python/python_version/domain.py index b624a06a..d7ce8fdd 100644 --- a/moe/optimal_learning/python/python_version/domain.py +++ b/moe/optimal_learning/python/python_version/domain.py @@ -92,8 +92,13 @@ def get_constraint_list(self, start_index=0): But in order for the constraints to access the correct index, the RepeatedDomain class has to signal which index the TensorProductDomain should start from, using the start_index optional parameter. + That is, RepeatedDomain deals with N d-dimensional points at once. Thus we need N*d constraints (one per + dimension, once per repeat). Additionally, instead of receiving points with shape (num_repeats, dim), COBYLA + requires that the points are flattened: (num_repeats*dim, ). Thus this method must know *where* in the + flattened-list it is writing to and reading from: signaled via ``start_index``. + :param start_index: the dimension this tensor product domain should start indexing from - :type start_index: integer (>= 0) + :type start_index: int >= 0 :return: a list of lambda functions corresponding to constraints :rtype: array of lambda functions with shape (dim * 2) @@ -124,7 +129,7 @@ def generate_uniform_random_points_in_domain(self, num_points, random_source=Non See python.geometry_utils.generate_latin_hypercube_points for more details. :param num_points: max number of points to generate - :type num_points: integer >= 0 + :type num_points: int >= 0 :param random_source: random source producing uniform random numbers (e.g., numpy.random.uniform) (UNUSED) :type random_source: callable yielding uniform random numbers in [0,1] :return: uniform random sampling of points from the domain diff --git a/moe/optimal_learning/python/python_version/gaussian_process.py b/moe/optimal_learning/python/python_version/gaussian_process.py index ac6e4198..c6708027 100644 --- a/moe/optimal_learning/python/python_version/gaussian_process.py +++ b/moe/optimal_learning/python/python_version/gaussian_process.py @@ -254,7 +254,7 @@ def _compute_grad_variance_of_points_per_point(self, points_to_sample, var_of_gr :param points_to_sample: num_to_sample points (in dim dimensions) being sampled from the GP :type points_to_sample: array of float64 with shape (num_to_sample, dim) :param var_of_grad: index of ``points_to_sample`` to be differentiated against - :type var_of_grad: integer in {0, .. ``num_to_sample``-1} + :type var_of_grad: int in {0, .. ``num_to_sample``-1} :return: grad_var: gradient of the variance matrix of this GP :rtype: array of float64 with shape (num_to_sample, num_to_sample, dim) @@ -322,7 +322,7 @@ def _compute_grad_cholesky_variance_of_points_per_point(self, points_to_sample, :param chol_var: the cholesky factorization (L) of the variance matrix; only the lower triangle is accessed :type chol_var: array of float64 with shape (num_to_sample, num_to_sample) :param var_of_grad: index of ``points_to_sample`` to be differentiated against - :type var_of_grad: integer in {0, .. ``num_to_sample``-1} + :type var_of_grad: int in {0, .. ``num_to_sample``-1} :return: grad_chol: gradient of the cholesky factorization of the variance matrix of this GP. ``grad_chol[j][i][d]`` is actually the gradients of ``var_{j,i}`` with respect to ``x_{k,d}``, the d-th dimension of the k-th entry of ``points_to_sample``, where diff --git a/moe/optimal_learning/python/python_version/optimization.py b/moe/optimal_learning/python/python_version/optimization.py index 4619a93d..9889d85f 100644 --- a/moe/optimal_learning/python/python_version/optimization.py +++ b/moe/optimal_learning/python/python_version/optimization.py @@ -312,7 +312,8 @@ class LBFGSBParameters(_BaseLBFGSBParameters): r"""Container to hold parameters that specify the behavior of L-BFGS-B. - Suggested values come from scipy documentation for scipy.optimize.fmin_l_bfgs_b. + Suggested values come from scipy documentation for ``scipy.optimize.fmin_l_bfgs_b``: + http://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.optimize.fmin_l_bfgs_b.html :ivar approx_grad: (*bool*) if true, BFGS will approximate the gradient :ivar max_func_evals: (*int > 0*) maximum number of objective function calls to make (suggest: 15000) @@ -325,9 +326,21 @@ class LBFGSBParameters(_BaseLBFGSBParameters): __slots__ = () + def scipy_kwargs(self): + """Return a dict that can be unpacked as kwargs to ``scipy.optimize.fmin_l_bfgs_b``. -# See ConstrainedDFOParameters (below) for docstring. -_BaseConstrainedDFOParameters = collections.namedtuple('_BaseConstrainedDFOParameters', [ + :return: kwargs for controlling the behavior of fmin_l_bfgs_b + :rtype: dict + + """ + out_dict = dict(self._asdict()) + out_dict['m'] = out_dict.pop('max_metric_correc') + out_dict['maxfun'] = out_dict.pop('max_func_evals') + return out_dict + + +# See COBYLAParameters (below) for docstring. +_BaseCOBYLAParameters = collections.namedtuple('_BaseCOBYLAParameters', [ 'rhobeg', 'rhoend', 'maxfun', @@ -335,12 +348,12 @@ class LBFGSBParameters(_BaseLBFGSBParameters): ]) -class ConstrainedDFOParameters(_BaseConstrainedDFOParameters): +class COBYLAParameters(_BaseCOBYLAParameters): r"""Container to hold parameters that specify the behavior of COBYLA. Suggested values come from scipy documentation for scipy.optimize.fmin_cobyla: - http://docs.scipy.org/doc/scipy-0.13.0/reference/generated/scipy.optimize.fmin_cobyla.html + http://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.optimize.fmin_cobyla.html :ivar rhobeg: (*float64 > 0.0*) reasonable initial changes to the variables (suggest: 1.0) :ivar rhoend: (*float64 > 0.0*) final accuracy in the optimization (not precisely guaranteed), which is a lower bound on the size of the trust region (suggest: 1.0e-4) @@ -351,6 +364,9 @@ class ConstrainedDFOParameters(_BaseConstrainedDFOParameters): __slots__ = () + # Return a dict that can be unpacked as kwargs to ``scipy.optimize.fmin_cobyla``. + scipy_kwargs = _BaseCOBYLAParameters._asdict + class NullOptimizer(OptimizerInterface): @@ -559,8 +575,8 @@ def optimize(self, random_starts=None, **kwargs): reported. It will otherwise report the overall best improvement (through io_container) as well as the result of every individual multistart run if desired (through function_values). - :param starting_points: points from which to multistart ``self.optimizer``; if None, points are chosen randomly - :type starting_points: array of float64 with shape (num_points, dim) or None + :param random_starts: points from which to multistart ``self.optimizer``; if None, points are chosen randomly + :type random_starts: array of float64 with shape (num_points, dim) or None :return: (best point found, objective function values at the end of each optimization run) :rtype: tuple: (array of float64 with shape (self.optimizer.dim), array of float64 with shape (self.num_multistarts)) @@ -590,10 +606,13 @@ class _ScipyOptimizerWrapper(OptimizerInterface): """Wrapper class to construct an optimizer from scipy optimization methods. - Requires the implementation of the :func:`~moe.optimal_learning.python.python_interface.optimization._ScipyOptimizerWrapper.get_scipy_optimizer_unshaped_point` method. + Requires the implementation of the :func:`~moe.optimal_learning.python.python_interface.optimization._ScipyOptimizerWrapper._optimize_core` method. """ + # Type of the optimization_parameters object, specified in subclass + optimization_parameters_type = None + def __init__(self, domain, optimizable, optimization_parameters): """Construct the optimizer. @@ -602,12 +621,17 @@ def __init__(self, domain, optimizable, optimization_parameters): :param optimizable: object representing the objective function being optimized :type optimizable: :class:`~moe.optimal_learning.python.interfaces.optimization_interface.OptimizableInterface` subclass :param optimization_parameters: parameters describing how to perform optimization (tolerances, iterations, etc.) - :type optimization_parameters: ``python_version.optimization.*Parameters`` object + :type optimization_parameters: ``python_version.optimization.*Parameters`` object, matching optimization_parameters_type """ self.domain = domain self.objective_function = optimizable - self.optimization_parameters = optimization_parameters + + if not isinstance(optimization_parameters, self.optimization_parameters_type): + raise TypeError('optimization_paramters is of type: {0:s}, expected {1:s}'.format(optimization_parameters.__class__, self.optimization_parameters_type)) + else: + self.optimization_parameters = optimization_parameters + self._num_points = 1 if hasattr(self.domain, 'num_repeats'): self._num_points = self.domain.num_repeats @@ -639,16 +663,14 @@ def decorated(point): return decorated def optimize(self, **kwargs): - """Shape the point returned by the get_scipy_optimizer_unshaped_point function. + """Perform optimization with :func:`~moe.optimal_learning.python.interfaces.optimization_interface._ScipyOptimizerWrapper._optimize_core` and shape the output point. - Calls the get_scipy_optimizer_unshaped_point method. - objective_function.current_point will be set to the optimal point found. + Calls the :func:`~moe.optimal_learning.python.interfaces.optimization_interface._ScipyOptimizerWrapper._optimize_core` method. + ``objective_function.current_point`` will be set to the optimal point found. """ - unshaped_point = self._get_scipy_optimizer_unshaped_point(**kwargs) + unshaped_point = self._optimize_core(**kwargs) - if not isinstance(unshaped_point, numpy.ndarray): - unshaped_point = unshaped_point[0] if self._num_points == 1: shaped_point = unshaped_point else: @@ -656,14 +678,14 @@ def optimize(self, **kwargs): self.objective_function.current_point = shaped_point @abstractmethod - def _get_scipy_optimizer_unshaped_point(self, **kwargs): + def _optimize_core(self, **kwargs): """Should return an unshaped point corresponding to the output of the optimizer function from scipy. - See :func:`~moe.optimal_learning.python.python_version.optimization.LBFGSOptimizer.get_scipy_optimizer_unshaped_point` or - :func:`~moe.optimal_learning.python.python_version.optimization.ConstrainedDFOOptimizer.get_scipy_optimizer_unshaped_point` function for examples. + See :func:`~moe.optimal_learning.python.python_version.optimization.LBFGSOptimizer._optimize_core` or + :func:`~moe.optimal_learning.python.python_version.optimization.COBYLAOptimizer._optimize_core` function for examples. :return: The unshaped optimal point from calling the scipy optimization method. - :rtype: array of float64 with shape (self._num_points, self.domain.dim) + :rtype: array of float64 with shape (self._num_points * self.domain.dim, ) """ pass @@ -687,6 +709,8 @@ class LBFGSBOptimizer(_ScipyOptimizerWrapper): """ + optimization_parameters_type = LBFGSBParameters + def __init__(self, domain, optimizable, optimization_parameters, num_random_samples=None): """Construct a LBFGSBOptimizer. @@ -700,8 +724,8 @@ def __init__(self, domain, optimizable, optimization_parameters, num_random_samp """ super(LBFGSBOptimizer, self).__init__(domain, optimizable, optimization_parameters) - def _get_scipy_optimizer_unshaped_point(self, **kwargs): - """Perform an L-BFGS-B optimization given the parameters in optimization_parameters.""" + def _optimize_core(self, **kwargs): + """Perform an L-BFGS-B optimization given the parameters in ``self.optimization_parameters``.""" domain_bounding_box = self.domain.get_bounding_box() domain_list = [(interval.min, interval.max) for interval in domain_bounding_box] domain_numpy = numpy.array(domain_list * self._num_points) @@ -712,16 +736,11 @@ def _get_scipy_optimizer_unshaped_point(self, **kwargs): x0=self.objective_function.current_point.flatten(), bounds=domain_numpy, fprime=self._scipy_decorator(self.objective_function.compute_grad_objective_function, **kwargs), - approx_grad=self.optimization_parameters.approx_grad, - factr=self.optimization_parameters.factr, - maxfun=self.optimization_parameters.max_func_evals, - m=self.optimization_parameters.max_metric_correc, - pgtol=self.optimization_parameters.pgtol, - epsilon=self.optimization_parameters.epsilon, + **self.optimization_parameters.scipy_kwargs() )[0] -class ConstrainedDFOOptimizer(_ScipyOptimizerWrapper): +class COBYLAOptimizer(_ScipyOptimizerWrapper): r"""Optimizes an objective function over the specified contraints with the COBYLA method. @@ -731,29 +750,28 @@ class ConstrainedDFOOptimizer(_ScipyOptimizerWrapper): """ + optimization_parameters_type = COBYLAParameters + def __init__(self, domain, optimizable, optimization_parameters): - """Construct a ConstrainedDFOOptimizer. + """Construct a COBYLAOptimizer. :param domain: the domain that this optimizer operates over :type domain: :class:`~moe.optimal_learning.python.interfaces.domain_interface.DomainInterface` subclass. Only supports TensorProductDomain for now. :param optimizable: object representing the objective function being optimized :type optimizable: :class:`~moe.optimal_learning.python.interfaces.optimization_interface.OptimizableInterface` subclass :param optimization_parameters: parameters describing how to perform optimization (tolerances, iterations, etc.) - :type optimization_parameters: :class:`~moe.optimal_learning.python.python_version.optimization.ConstrainedDFOParameters` object + :type optimization_parameters: :class:`~moe.optimal_learning.python.python_version.optimization.COBYLAParameters` object """ - super(ConstrainedDFOOptimizer, self).__init__(domain, optimizable, optimization_parameters) + super(COBYLAOptimizer, self).__init__(domain, optimizable, optimization_parameters) - def _get_scipy_optimizer_unshaped_point(self, **kwargs): - """Perform a COBYLA optimization given the parameters in optimization_parameters.""" - # Parameters defined above in :class:`~moe.optimal_learning.python.python_version.optimization.ConstrainedDFOParameters` class. + def _optimize_core(self, **kwargs): + """Perform a COBYLA optimization given the parameters in ``self.optimization_parameters``.""" + # Parameters defined above in :class:`~moe.optimal_learning.python.python_version.optimization.COBYLAParameters` class. return scipy.optimize.fmin_cobyla( func=self._scipy_decorator(self.objective_function.compute_objective_function, **kwargs), x0=self.objective_function.current_point.flatten(), cons=self.domain.get_constraint_list(), - rhobeg=self.optimization_parameters.rhobeg, - rhoend=self.optimization_parameters.rhoend, - maxfun=self.optimization_parameters.maxfun, - catol=self.optimization_parameters.catol, disp=0, # Suppresses output from the routine. + **self.optimization_parameters.scipy_kwargs() ) diff --git a/moe/optimal_learning/python/repeated_domain.py b/moe/optimal_learning/python/repeated_domain.py index 718e1a14..638cd2b5 100644 --- a/moe/optimal_learning/python/repeated_domain.py +++ b/moe/optimal_learning/python/repeated_domain.py @@ -82,14 +82,17 @@ def get_bounding_box(self): def get_constraint_list(self): """Return a list of lambda functions expressing the domain bounds as linear constraints. Used by COBYLA. + Calls ``self._domain.get_constraint_list()`` for each repeat, writing the results sequentially. + So output[0:2*dim] is from the first repeated domain, output[2*dim:4*dim] is from the second, etc. + :return: a list of lambda functions corresponding to constraints - :rtype: array of lambda functions with shape (dim * 2) + :rtype: array of lambda functions with shape (num_repeats * dim * 2) """ constraints = [] for i in xrange(self.num_repeats): # Using start_index, start each domain at the correct index when flattening out points in COBYLA. - constraints.extend(self._domain.get_constraint_list(start_index=self._domain.dim * i)) + constraints.extend(self._domain.get_constraint_list(start_index=self.dim * i)) return constraints def generate_random_point_in_domain(self, random_source=None): diff --git a/moe/tests/optimal_learning/python/python_version/optimization_test.py b/moe/tests/optimal_learning/python/python_version/optimization_test.py index a86ab3d0..f8654fc6 100644 --- a/moe/tests/optimal_learning/python/python_version/optimization_test.py +++ b/moe/tests/optimal_learning/python/python_version/optimization_test.py @@ -7,7 +7,7 @@ from moe.optimal_learning.python.geometry_utils import ClosedInterval from moe.optimal_learning.python.interfaces.optimization_interface import OptimizableInterface from moe.optimal_learning.python.python_version.domain import TensorProductDomain -from moe.optimal_learning.python.python_version.optimization import multistart_optimize, LBFGSBParameters, GradientDescentParameters, NullOptimizer, GradientDescentOptimizer, MultistartOptimizer, LBFGSBOptimizer, ConstrainedDFOOptimizer, ConstrainedDFOParameters +from moe.optimal_learning.python.python_version.optimization import multistart_optimize, LBFGSBParameters, GradientDescentParameters, NullOptimizer, GradientDescentOptimizer, MultistartOptimizer, LBFGSBOptimizer, COBYLAOptimizer, COBYLAParameters from moe.tests.optimal_learning.python.optimal_learning_test_case import OptimalLearningTestCase @@ -209,7 +209,7 @@ def base_setup(cls): rhobeg = 1.0 rhoend = numpy.finfo(numpy.float64).eps catol = 2.0e-13 - cls.COBYLA_parameters = ConstrainedDFOParameters( + cls.COBYLA_parameters = COBYLAParameters( rhobeg, rhoend, maxfun, @@ -265,15 +265,14 @@ def test_gradient_descent_optimizer_constrained(self): def test_multistarted_gradient_descent_optimizer_crippled_start(self): """Check that multistarted GD is finding the best result from GD.""" # Only allow 1 GD iteration. - gd_parameters_crippled = GradientDescentParameters( - 1, - 1, - self.gd_parameters.num_steps_averaged, - self.gd_parameters.gamma, - self.gd_parameters.pre_mult, - self.gd_parameters.max_relative_change, - self.gd_parameters.tolerance, - ) + max_num_steps = 1 + max_num_restarts = 1 + + param_dict = self.gd_parameters._asdict() + param_dict['max_num_steps'] = max_num_steps + param_dict['max_num_restarts'] = max_num_restarts + gd_parameters_crippled = GradientDescentParameters(**param_dict) + gradient_descent_optimizer_crippled = GradientDescentOptimizer(self.domain, self.polynomial, gd_parameters_crippled) num_points = 15 @@ -340,7 +339,7 @@ def multistarted_optimizer_test(self, optimizer): def test_cobyla_optimizer(self): """Test if COBYLA can optimize a simple objective function.""" - constrained_optimizer = ConstrainedDFOOptimizer(self.domain, self.polynomial, self.COBYLA_parameters) + constrained_optimizer = COBYLAOptimizer(self.domain, self.polynomial, self.COBYLA_parameters) self.optimizer_test(constrained_optimizer) def test_bfgs_optimizer(self): @@ -355,7 +354,7 @@ def test_gradient_descent_optimizer(self): def test_cobyla_multistarted_optimizer(self): """Test if COBYLA can optimize a "hard" objective function with multistarts.""" - constrained_optimizer = ConstrainedDFOOptimizer(self.large_domain, self.polynomial, self.COBYLA_parameters) + constrained_optimizer = COBYLAOptimizer(self.large_domain, self.polynomial, self.COBYLA_parameters) self.multistarted_optimizer_test(constrained_optimizer) def test_bfgs_multistarted_optimizer(self): @@ -376,15 +375,9 @@ def test_gradient_descent_optimizer_with_averaging(self): """ num_steps_averaged = self.gd_parameters.max_num_steps * 3 / 4 - gd_parameters_averaging = GradientDescentParameters( - self.gd_parameters.max_num_steps, - self.gd_parameters.max_num_restarts, - num_steps_averaged, - self.gd_parameters.gamma, - self.gd_parameters.pre_mult, - self.gd_parameters.max_relative_change, - self.gd_parameters.tolerance, - ) + param_dict = self.gd_parameters._asdict() + param_dict['num_steps_averaged'] = num_steps_averaged + gd_parameters_averaging = GradientDescentParameters(**param_dict) gradient_descent_optimizer = GradientDescentOptimizer(self.domain, self.polynomial, gd_parameters_averaging) self.optimizer_test(gradient_descent_optimizer, tolerance=2.0e-10)