Updates to the library:

- Refactored initial design code
- Fixed bug in arguments manager
- Fixed bug in modular opt
- Added more tests

GPyOpt/__init__.py

@@ -4,9 +4,6 @@
 import warnings
 warnings.filterwarnings("ignore", category=DeprecationWarning)
 
-import matplotlib as mpl
-mpl.use('Agg')
-
 from GPyOpt.core.task.space import Design_space
 from . import core
 from . import methods
@@ -19,5 +16,4 @@
 from . import objective_examples
 from . import objective_examples as fmodels
 
-
 from .__version__ import __version__

GPyOpt/core/task/space.py

@@ -241,7 +241,7 @@ def model_to_objective(self, x_model):
 
         return x_objective
 
-    def has_constrains(self):
+    def has_constraints(self):
         """
         Checks if the problem has constrains. Note that the coordinates of the constrains are defined
         in terms of the model inputs and not in terms of the objective inputs. This means that if bandit or
@@ -262,7 +262,10 @@ def get_bounds(self):
 
         return bounds
 
-    def _has_continuous(self):
+    def has_continuous(self):
+        """
+        Returns `true` if the space contains at least one continuous variable, and `false` otherwise
+        """
         return any(v.is_continuous() for v in self.space)
 
     def _has_bandit(self):

GPyOpt/experiment_design/__init__.py

@@ -0,0 +1,20 @@
+from .base import ExperimentDesign
+from .grid_design import GridDesign
+from .latin_design import LatinDesign
+from .random_design import RandomDesign
+from .sobol_design import SobolDesign
+
+def initial_design(design_name, space, init_points_count):
+    design = None
+    if design_name == 'random':
+        design = RandomDesign(space)
+    elif design_name == 'sobol':
+        design = SobolDesign(space)
+    elif design_name == 'grid':
+        design = GridDesign(space)
+    elif design_name == 'latin':
+        design = LatinDesign(space)
+    else:
+        raise ValueError('Unknown design type: ' + design_name)
+
+    return design.get_samples(init_points_count)

GPyOpt/experiment_design/base.py

@@ -0,0 +1,9 @@
+class ExperimentDesign(object):
+    """
+    Base class for all experiment designs
+    """
+    def __init__(self, space):
+        self.space = space
+
+    def get_samples(self, init_points_count):
+        raise NotImplementedError("Subclasses should implement this method.")

GPyOpt/experiment_design/grid_design.py

@@ -0,0 +1,70 @@
+import numpy as np
+
+from ..core.errors import InvalidConfigError
+
+from .base import ExperimentDesign
+from .random_design import RandomDesign
+
+class GridDesign(ExperimentDesign):
+    """
+    Grid experiment design.
+    Uses random design for non-continuous variables, and square grid for continuous ones
+    """
+
+    def __init__(self, space):
+        if space.has_constraints():
+            raise InvalidConfigError('Sampling with constrains is not allowed by grid design')
+        super(GridDesign, self).__init__(space)
+
+    def _adjust_init_points_count(self, init_points_count):
+        # TODO: log this
+        print('Note: in grid designs the total number of generated points is the smallest closest integer of n^d to the selected amount of points')
+        continuous_dims = len(self.space.get_continuous_dims())
+        self.data_per_dimension = iroot(continuous_dims, init_points_count)
+        return self.data_per_dimension**continuous_dims
+
+    def get_samples(self, init_points_count):
+        """
+        This method may return less points than requested.
+        The total number of generated points is the smallest closest integer of n^d to the selected amount of points.
+        """
+
+        init_points_count = self._adjust_init_points_count(init_points_count)
+        samples = np.empty((init_points_count, self.space.dimensionality))
+
+        # Use random design to fill non-continuous variables
+        random_design = RandomDesign(self.space)
+        random_design.fill_noncontinous_variables(samples)
+
+        if self.space.has_continuous():
+            X_design = multigrid(self.space.get_continuous_bounds(), self.data_per_dimension)
+            samples[:,self.space.get_continuous_dims()] = X_design
+
+        return samples
+
+# Computes integer root
+# The greatest integer whose k-th power is less than or equal to n
+# That is the greatest x such that x^k <= n
+def iroot(k, n):
+    # Implements Newton Iroot algorithm
+    # Details can be found here: https://www.akalin.com/computing-iroot
+    # In a nutshell, it constructs a decreasing number series
+    # that is guaranteed to terminate at the required integer root
+    u, s = n, n+1
+    while u < s:
+        s = u
+        t = (k-1) * s + n // pow(s, k-1)
+        u = t // k
+    return s
+
+def multigrid(bounds, points_count):
+    """
+    Generates a multidimensional lattice
+    :param bounds: box constrains
+    :param points_count: number of points per dimension.
+    """
+    if len(bounds)==1:
+        return np.linspace(bounds[0][0], bounds[0][1], points_count).reshape(points_count, 1)
+    x_grid_rows = np.meshgrid(*[np.linspace(b[0], b[1], points_count) for b in bounds])
+    x_grid_columns = np.vstack([x.flatten(order='F') for x in x_grid_rows]).T
+    return x_grid_columns

GPyOpt/experiment_design/latin_design.py

@@ -0,0 +1,38 @@
+import numpy as np
+
+from ..core.errors import InvalidConfigError
+
+from .base import ExperimentDesign
+from .random_design import RandomDesign
+
+class LatinDesign(ExperimentDesign):
+    """
+    Latin experiment design.
+    Uses random design for non-continuous variables, and latin hypercube for continuous ones
+    """
+    def __init__(self, space):
+        if space.has_constraints():
+            raise InvalidConfigError('Sampling with constrains is not allowed by latin design')
+        super(LatinDesign, self).__init__(space)
+
+    def get_samples(self, init_points_count):
+        samples = np.empty((init_points_count, self.space.dimensionality))
+
+        # Use random design to fill non-continuous variables
+        random_design = RandomDesign(self.space)
+        random_design.fill_noncontinous_variables(samples)
+
+        if self.space.has_continuous():
+            bounds = self.space.get_continuous_bounds()
+            lower_bound = np.asarray(bounds)[:,0].reshape(1, len(bounds))
+            upper_bound = np.asarray(bounds)[:,1].reshape(1, len(bounds))
+            diff = upper_bound - lower_bound
+
+            from pyDOE import lhs
+            X_design_aux = lhs(len(self.space.get_continuous_bounds()), init_points_count, criterion='center')
+            I = np.ones((X_design_aux.shape[0], 1))
+            X_design = np.dot(I, lower_bound) + X_design_aux * np.dot(I, diff)
+
+            samples[:, self.space.get_continuous_dims()] = X_design
+
+        return samples

GPyOpt/experiment_design/random_design.py

@@ -0,0 +1,75 @@
+import numpy as np
+
+from .base import ExperimentDesign
+from ..core.task.variables import BanditVariable, DiscreteVariable, CategoricalVariable
+
+
+class RandomDesign(ExperimentDesign):
+    """
+    Random experiment design.
+    Random values for all variables within the given bounds.
+    """
+    def __init__(self, space):
+        super(RandomDesign, self).__init__(space)
+
+    def get_samples(self, init_points_count):
+        if self.space.has_constraints():
+            return self.get_samples_with_constraints(init_points_count)
+        else:
+            return self.get_samples_without_constraints(init_points_count)
+
+    def get_samples_with_constraints(self, init_points_count):
+        """
+        Draw random samples and only save those that satisfy constrains
+        Finish when required number of samples is generated
+        """
+        samples = np.empty((0, self.space.dimensionality))
+
+        while samples.shape[0] < init_points_count:
+            domain_samples = self.get_samples_without_constraints(init_points_count)
+            valid_indices = (self.space.indicator_constraints(domain_samples) == 1).flatten()
+            if sum(valid_indices) > 0:
+                valid_samples = domain_samples[valid_indices,:]
+                samples = np.vstack((samples,valid_samples))
+
+        return samples[0:init_points_count,:]
+
+    def fill_noncontinous_variables(self, samples):
+        """
+        Fill sample values to non-continuous variables in place
+        """
+        init_points_count = samples.shape[0]
+
+        for (idx, var) in enumerate(self.space.space_expanded):
+            if isinstance(var, DiscreteVariable) or isinstance(var, CategoricalVariable) :
+                sample_var = np.atleast_2d(np.random.choice(var.domain, init_points_count))
+                samples[:,idx] = sample_var.flatten()
+
+            # sample in the case of bandit variables
+            elif isinstance(var, BanditVariable):
+                idx_samples = np.random.randint(var.domain.shape[0], size=init_points_count)
+                samples[:, idx] = var.domain[idx_samples,:]
+
+
+    def get_samples_without_constraints(self, init_points_count):
+        samples = np.empty((init_points_count, self.space.dimensionality))
+
+        self.fill_noncontinous_variables(samples)
+
+        if self.space.has_continuous():
+            X_design = samples_multidimensional_uniform(self.space.get_continuous_bounds(), init_points_count)
+            samples[:, self.space.get_continuous_dims()] = X_design
+
+        return samples
+
+def samples_multidimensional_uniform(bounds, points_count):
+    """
+    Generates a multidimensional grid uniformly distributed.
+    :param bounds: tuple defining the box constrains.
+    :points_count: number of data points to generate.
+    """
+    dim = len(bounds)
+    Z_rand = np.zeros(shape=(points_count, dim))
+    for k in range(0,dim):
+        Z_rand[:,k] = np.random.uniform(low=bounds[k][0], high=bounds[k][1], size=points_count)
+    return Z_rand

GPyOpt/experiment_design/sobol_design.py

@@ -0,0 +1,35 @@
+import numpy as np
+
+from ..core.errors import InvalidConfigError
+
+from .base import ExperimentDesign
+from .random_design import RandomDesign
+
+class SobolDesign(ExperimentDesign):
+    """
+    Sobol experiment design.
+    Uses random design for non-continuous variables, and Sobol sequence for continuous ones
+    """
+    def __init__(self, space):
+        if space.has_constraints():
+            raise InvalidConfigError('Sampling with constrains is not allowed by Sobol design')
+        super(SobolDesign, self).__init__(space)
+
+    def get_samples(self, init_points_count):
+        samples = np.empty((init_points_count, self.space.dimensionality))
+
+        # Use random design to fill non-continuous variables
+        random_design = RandomDesign(self.space)
+        random_design.fill_noncontinous_variables(samples)
+
+        if self.space.has_continuous():
+            bounds = self.space.get_continuous_bounds()
+            lower_bound = np.asarray(bounds)[:,0].reshape(1,len(bounds))
+            upper_bound = np.asarray(bounds)[:,1].reshape(1,len(bounds))
+            diff = upper_bound-lower_bound
+
+            from sobol_seq import i4_sobol_generate
+            X_design = np.dot(i4_sobol_generate(len(self.space.get_continuous_bounds()),init_points_count),np.diag(diff.flatten()))[None,:] + lower_bound
+            samples[:, self.space.get_continuous_dims()] = X_design
+
+        return samples

GPyOpt/interface/driver.py

@@ -108,7 +108,7 @@ def run(self):
         acq = self._get_acquisition(model, space)
         acq_eval = self._get_acq_evaluator(acq)
 
-        from ..util.stats import initial_design
+        from ..experiment_design import initial_design
         X_init = initial_design(self.config['initialization']['type'], space, self.config['initialization']['num-eval'])
 
         from ..methods import ModularBayesianOptimization

GPyOpt/methods/bayesian_optimization.py

@@ -7,7 +7,7 @@
 from ..core.task.space import Design_space, bounds_to_space
 from ..core.task.objective import SingleObjective
 from ..core.task.cost import CostModel
-from ..util.general import initial_design
+from ..experiment_design import initial_design
 from ..util.arguments_manager import ArgumentsManager
 from ..core.evaluators import Sequential, RandomBatch, LocalPenalization, ThompsonBatch
 from ..models.gpmodel import GPModel, GPModel_MCMC

GPyOpt/methods/modular_bayesian_optimization.py

@@ -38,6 +38,3 @@ def __init__(self, model, space, objective, acquisition, evaluator, X_init, Y_in
                                                             normalize_Y            = normalize_Y,
                                                             model_update_interval  = model_update_interval,
                                                             de_duplication         = de_duplication)
-
-        # --- Initialize everything
-        self.run_optimization(0)

GPyOpt/optimization/anchor_points_generator.py

@@ -2,7 +2,7 @@
 # Licensed under the BSD 3-clause license (see LICENSE.txt)
 
 import numpy as np
-from ..util.general import initial_design
+from ..experiment_design import initial_design
 from ..core.errors import FullyExploredOptimizationDomainError
 from ..core.task.space import Design_space
 

GPyOpt/testing/core_tests/test_design_space.py

@@ -88,7 +88,7 @@ def test_create_constraints(self):
         design_space = Design_space(space, constraints=constraints)
 
         self.assertEqual(len(design_space.space_expanded), 2)
-        self.assertTrue(design_space.has_constrains())
+        self.assertTrue(design_space.has_constraints())
 
     def test_bounds(self):
         space = [

GPyOpt/testing/functional_tests/test_constrains.py

@@ -73,7 +73,7 @@ def setUp(self):
 
 
         feasible_region = GPyOpt.Design_space(space = self.problem_config['domain'], constraints = self.problem_config['constrains'])
-        self.f_inits = GPyOpt.util.general.initial_design('random', feasible_region, 5)
+        self.f_inits = GPyOpt.experiment_design.initial_design('random', feasible_region, 5)
         self.f_inits = self.f_inits.reshape(n_inital_design, input_dim)
 
     def test_run(self):

GPyOpt/testing/functional_tests/test_context.py

@@ -96,7 +96,7 @@ def setUp(self):
             }
 
         feasible_region = GPyOpt.Design_space(space = self.problem_config['domain'], constraints = self.problem_config['constrains'])
-        self.f_inits = GPyOpt.util.general.initial_design('random', feasible_region, 5)
+        self.f_inits = GPyOpt.experiment_design.initial_design('random', feasible_region, 5)
         self.f_inits = self.f_inits.reshape(n_inital_design, input_dim)
 
     def test_run(self):

GPyOpt/testing/functional_tests/test_context_and_parallelization.py

@@ -130,7 +130,7 @@ def setUp(self):
             }
 
         feasible_region = GPyOpt.Design_space(space = self.problem_config['domain'], constraints = self.problem_config['constrains'])
-        self.f_inits = GPyOpt.util.general.initial_design('random', feasible_region, 5)
+        self.f_inits = GPyOpt.experiment_design.initial_design('random', feasible_region, 5)
         self.f_inits = self.f_inits.reshape(n_inital_design, input_dim)
 
     def test_run(self):

GPyOpt/testing/functional_tests/test_context_manager.py

@@ -2,7 +2,7 @@
 import numpy as np
 
 from GPyOpt.core.task.space import Design_space
-from GPyOpt.util.general import initial_design
+from GPyOpt.experiment_design import initial_design
 from GPyOpt.optimization.acquisition_optimizer import ContextManager
 
 class TestContextManager(unittest.TestCase):

GPyOpt/testing/functional_tests/test_duplicate_manager.py

@@ -2,7 +2,7 @@
 import numpy as np
 
 from GPyOpt.core.task.space import Design_space
-from GPyOpt.util.general import initial_design
+from GPyOpt.experiment_design import initial_design
 from GPyOpt.util.duplicate_manager import DuplicateManager
 
 class TestDuplicateManager(unittest.TestCase):

GPyOpt/testing/functional_tests/test_input_warped_gp.py

@@ -75,7 +75,7 @@ def setUp(self):
 
 
         feasible_region = GPyOpt.Design_space(space = self.problem_config['domain'], constraints = self.problem_config['constrains'])
-        self.f_inits = GPyOpt.util.general.initial_design('random', feasible_region, 5)
+        self.f_inits = GPyOpt.experiment_design.initial_design('random', feasible_region, 5)
         self.f_inits = self.f_inits.reshape(n_inital_design, input_dim)
 
     def test_run(self):