From e9573c1ef9d84776f2f74862622de0a0bc678605 Mon Sep 17 00:00:00 2001 From: Prasanna Balaprakash Date: Mon, 18 Oct 2021 04:31:35 -0500 Subject: [PATCH] support for truncated normal distribution (#188) * adding trucated normal support * adding trucated normal support * updating the documentation * edited setup.py with scipy * minor editions to improve integration of truncated normal * removing dublicated definition of public parameters lower/upper * improving documentation consistency, adding tests for quantization and border case, correcting representation * Update setup.py Co-authored-by: Matthias Feurer * fixed NormalInteger with truncated normal conversion to integer, added test cases * adaptation of get_neighbors with tests * pre-commit tests passing Co-authored-by: Deathn0t Co-authored-by: Matthias Feurer --- ConfigSpace/hyperparameters.pyx | 197 ++++++++++++++++++++++++++------ setup.py | 2 +- test/test_hyperparameters.py | 71 ++++++++++++ 3 files changed, 234 insertions(+), 36 deletions(-) diff --git a/ConfigSpace/hyperparameters.pyx b/ConfigSpace/hyperparameters.pyx index 2730dedb..4bd56a25 100644 --- a/ConfigSpace/hyperparameters.pyx +++ b/ConfigSpace/hyperparameters.pyx @@ -35,6 +35,7 @@ from collections import OrderedDict, Counter from typing import List, Any, Dict, Union, Set, Tuple, Optional import numpy as np +from scipy.stats import truncnorm cimport numpy as np @@ -200,9 +201,9 @@ cdef class Constant(Hyperparameter): Additionally, it defines the __ne__() as stated in the documentation from python: - By default, object implements __eq__() by using is, returning NotImplemented - in the case of a false comparison: True if x is y else NotImplemented. - For __ne__(), by default it delegates to __eq__() and inverts the result + By default, object implements __eq__() by using is, returning NotImplemented + in the case of a false comparison: True if x is y else NotImplemented. + For __ne__(), by default it delegates to __eq__() and inverts the result unless it is NotImplemented. """ @@ -298,9 +299,9 @@ cdef class NumericalHyperparameter(Hyperparameter): Additionally, it defines the __ne__() as stated in the documentation from python: - By default, object implements __eq__() by using is, returning NotImplemented - in the case of a false comparison: True if x is y else NotImplemented. - For __ne__(), by default it delegates to __eq__() and inverts the result + By default, object implements __eq__() by using is, returning NotImplemented + in the case of a false comparison: True if x is y else NotImplemented. + For __ne__(), by default it delegates to __eq__() and inverts the result unless it is NotImplemented. """ @@ -432,7 +433,7 @@ cdef class UniformFloatHyperparameter(FloatHyperparameter): ---------- name : str Name of the hyperparameter, with which it can be accessed - lower : int, floor + lower : int, float Lower bound of a range of values from which the hyperparameter will be sampled upper : int, float Upper bound @@ -610,6 +611,8 @@ cdef class NormalFloatHyperparameter(FloatHyperparameter): def __init__(self, name: str, mu: Union[int, float], sigma: Union[int, float], default_value: Union[None, float] = None, q: Union[int, float, None] = None, log: bool = False, + lower: Optional[Union[float, int]] = None, + upper: Optional[Union[float, int]] = None, meta: Optional[Dict] = None) -> None: r""" A float hyperparameter. @@ -643,6 +646,10 @@ cdef class NormalFloatHyperparameter(FloatHyperparameter): log : bool, optional If ``True``, the values of the hyperparameter will be sampled on a logarithmic scale. Default to ``False`` + lower : int, float, optional + Lower bound of a range of values from which the hyperparameter will be sampled + upper : int, float, optional + Upper bound of a range of values from which the hyperparameter will be sampled meta : Dict, optional Field for holding meta data provided by the user. Not used by the configuration space. @@ -655,11 +662,58 @@ cdef class NormalFloatHyperparameter(FloatHyperparameter): self.default_value = self.check_default(default_value) self.normalized_default_value = self._inverse_transform(self.default_value) + if (lower is not None) ^ (upper is not None): + raise ValueError("Only one bound was provided when both lower and upper bounds must be provided.") + + if lower is not None and upper is not None: + self.lower = float(lower) + self.upper = float(upper) + + if self.lower >= self.upper: + raise ValueError("Upper bound %f must be larger than lower bound " + "%f for hyperparameter %s" % + (self.upper, self.lower, name)) + elif log and self.lower <= 0: + raise ValueError("Negative lower bound (%f) for log-scale " + "hyperparameter %s is forbidden." % + (self.lower, name)) + + self.default_value = self.check_default(default_value) + + if self.log: + if self.q is not None: + lower = self.lower - (np.float64(self.q) / 2. + 0.0001) + upper = self.upper + (np.float64(self.q) / 2. - 0.0001) + else: + lower = self.lower + upper = self.upper + self._lower = np.log(lower) + self._upper = np.log(upper) + else: + if self.q is not None: + self._lower = self.lower - (self.q / 2. + 0.0001) + self._upper = self.upper + (self.q / 2. - 0.0001) + else: + self._lower = self.lower + self._upper = self.upper + if self.q is not None: + # There can be weird rounding errors, so we compare the result against self.q, see + # In [13]: 2.4 % 0.2 + # Out[13]: 0.1999999999999998 + if np.round((self.upper - self.lower) % self.q, 10) not in (0, self.q): + raise ValueError( + 'Upper bound (%f) - lower bound (%f) must be a multiple of q (%f)' + % (self.upper, self.lower, self.q) + ) + def __repr__(self) -> str: repr_str = io.StringIO() - repr_str.write("%s, Type: NormalFloat, Mu: %s Sigma: %s, Default: %s" % - (self.name, repr(self.mu), repr(self.sigma), - repr(self.default_value))) + + if self.lower is None or self.upper is None: + repr_str.write("%s, Type: NormalFloat, Mu: %s Sigma: %s, Default: %s" % (self.name, repr(self.mu), repr(self.sigma), repr(self.default_value))) + else: + repr_str.write("%s, Type: NormalFloat, Mu: %s Sigma: %s, Range: [%s, %s], Default: %s" % (self.name, repr(self.mu), repr(self.sigma), repr(self.lower), repr(self.upper), repr(self.default_value))) + if self.log: repr_str.write(", on log-scale") if self.q is not None: @@ -674,9 +728,9 @@ cdef class NormalFloatHyperparameter(FloatHyperparameter): Additionally, it defines the __ne__() as stated in the documentation from python: - By default, object implements __eq__() by using is, returning NotImplemented - in the case of a false comparison: True if x is y else NotImplemented. - For __ne__(), by default it delegates to __eq__() and inverts the result + By default, object implements __eq__() by using is, returning NotImplemented + in the case of a false comparison: True if x is y else NotImplemented. + For __ne__(), by default it delegates to __eq__() and inverts the result unless it is NotImplemented. """ @@ -689,7 +743,9 @@ cdef class NormalFloatHyperparameter(FloatHyperparameter): self.mu == other.mu and self.sigma == other.sigma and self.log == other.log and - self.q == other.q + self.q == other.q and + self.lower == other.lower and + self.upper == other.upper ) def __copy__(self): @@ -700,6 +756,8 @@ cdef class NormalFloatHyperparameter(FloatHyperparameter): sigma=self.sigma, log=self.log, q=self.q, + lower=self.lower, + upper=self.upper, meta=self.meta ) @@ -707,9 +765,16 @@ cdef class NormalFloatHyperparameter(FloatHyperparameter): return hash((self.name, self.mu, self.sigma, self.log, self.q)) def to_uniform(self, z: int = 3) -> 'UniformFloatHyperparameter': + if self.lower is None or self.upper is None: + lb = self.mu - (z * self.sigma) + ub = self.mu + (z * self.sigma) + else: + lb = self.lower + ub = self.upper + return UniformFloatHyperparameter(self.name, - self.mu - (z * self.sigma), - self.mu + (z * self.sigma), + lb, + ub, default_value=int( np.round(self.default_value, 0)), q=self.q, log=self.log) @@ -740,9 +805,19 @@ cdef class NormalFloatHyperparameter(FloatHyperparameter): def _sample(self, rs: np.random.RandomState, size: Optional[int] = None ) -> Union[np.ndarray, float]: - mu = self.mu - sigma = self.sigma - return rs.normal(mu, sigma, size=size) + + if self.lower == None: + mu = self.mu + sigma = self.sigma + return rs.normal(mu, sigma, size=size) + else: + mu = self.mu + sigma = self.sigma + lower = self.lower + upper = self.upper + a = (self.lower - mu) / sigma + b = (upper - mu) / sigma + return truncnorm.rvs(a, b, loc=mu, scale=sigma, size=size, random_state=rs) cpdef np.ndarray _transform_vector(self, np.ndarray vector): if np.isnan(vector).any(): @@ -774,7 +849,12 @@ cdef class NormalFloatHyperparameter(FloatHyperparameter): transform: bool = False) -> List[float]: neighbors = [] for i in range(number): - neighbors.append(rs.normal(value, self.sigma)) + new_value = rs.normal(value, self.sigma) + + if self.lower is not None and self.upper is not None: + new_value = min(max(new_value, self.lower), self.upper) + + neighbors.append(new_value) return neighbors @@ -1034,9 +1114,12 @@ cdef class NormalIntegerHyperparameter(IntegerHyperparameter): cdef public sigma cdef nfhp + def __init__(self, name: str, mu: int, sigma: Union[int, float], default_value: Union[int, None] = None, q: Union[None, int] = None, log: bool = False, + lower: Optional[int] = None, + upper: Optional[int] = None, meta: Optional[Dict] = None) -> None: r""" An integer hyperparameter. @@ -1071,12 +1154,17 @@ cdef class NormalIntegerHyperparameter(IntegerHyperparameter): log : bool, optional If ``True``, the values of the hyperparameter will be sampled on a logarithmic scale. Defaults to ``False`` + lower : int, float, optional + Lower bound of a range of values from which the hyperparameter will be sampled + upper : int, float, optional + Upper bound of a range of values from which the hyperparameter will be sampled meta : Dict, optional Field for holding meta data provided by the user. Not used by the configuration space. """ super(NormalIntegerHyperparameter, self).__init__(name, default_value, meta) + self.mu = mu self.sigma = sigma @@ -1097,20 +1185,43 @@ cdef class NormalIntegerHyperparameter(IntegerHyperparameter): self.default_value = self.check_default(default_value) + if (lower is not None) ^ (upper is not None): + raise ValueError("Only one bound was provided when both lower and upper bounds must be provided.") + + if lower is not None and upper is not None: + self.upper = self.check_int(upper, "upper") + self.lower = self.check_int(lower, "lower") + if self.lower >= self.upper: + raise ValueError("Upper bound %d must be larger than lower bound " + "%d for hyperparameter %s" % + (self.lower, self.upper, name)) + elif log and self.lower <= 0: + raise ValueError("Negative lower bound (%d) for log-scale " + "hyperparameter %s is forbidden." % + (self.lower, name)) + self.lower = lower + self.upper = upper + + self.nfhp = NormalFloatHyperparameter(self.name, self.mu, self.sigma, log=self.log, q=self.q, + lower=self.lower, + upper=self.upper, default_value=self.default_value) self.normalized_default_value = self._inverse_transform(self.default_value) def __repr__(self) -> str: repr_str = io.StringIO() - repr_str.write("%s, Type: NormalInteger, Mu: %s Sigma: %s, Default: " - "%s" % (self.name, repr(self.mu), - repr(self.sigma), repr(self.default_value))) + + if self.lower is None or self.upper is None: + repr_str.write("%s, Type: NormalInteger, Mu: %s Sigma: %s, Default: %s" % (self.name, repr(self.mu), repr(self.sigma), repr(self.default_value))) + else: + repr_str.write("%s, Type: NormalInteger, Mu: %s Sigma: %s, Range: [%s, %s], Default: %s" % (self.name, repr(self.mu), repr(self.sigma), repr(self.lower), repr(self.upper), repr(self.default_value))) + if self.log: repr_str.write(", on log-scale") if self.q is not None: @@ -1125,9 +1236,9 @@ cdef class NormalIntegerHyperparameter(IntegerHyperparameter): Additionally, it defines the __ne__() as stated in the documentation from python: - By default, object implements __eq__() by using is, returning NotImplemented - in the case of a false comparison: True if x is y else NotImplemented. - For __ne__(), by default it delegates to __eq__() and inverts the result + By default, object implements __eq__() by using is, returning NotImplemented + in the case of a false comparison: True if x is y else NotImplemented. + For __ne__(), by default it delegates to __eq__() and inverts the result unless it is NotImplemented. """ @@ -1139,7 +1250,9 @@ cdef class NormalIntegerHyperparameter(IntegerHyperparameter): self.mu == other.mu and self.sigma == other.sigma and self.log == other.log and - self.q == other.q + self.q == other.q and + self.lower == other.lower and + self.upper == other.upper ) def __hash__(self): @@ -1153,14 +1266,23 @@ cdef class NormalIntegerHyperparameter(IntegerHyperparameter): sigma=self.sigma, log=self.log, q=self.q, + lower=self.lower, + upper=self.upper, meta=self.meta ) # todo check if conversion should be done in initiation call or inside class itsel def to_uniform(self, z: int = 3) -> 'UniformIntegerHyperparameter': + if self.lower is None or self.upper is None: + lb = np.round(int(self.mu - (z * self.sigma))) + ub = np.round(int(self.mu + (z * self.sigma))) + else: + lb = self.lower + ub = self.upper + return UniformIntegerHyperparameter(self.name, - np.round(int(self.mu - (z * self.sigma))), - np.round(int(self.mu + (z * self.sigma))), + lb, + ub, default_value=self.default_value, q=self.q, log=self.log) @@ -1220,6 +1342,11 @@ cdef class NormalIntegerHyperparameter(IntegerHyperparameter): new_value = rs.normal(value, self.sigma) int_value = self._transform(value) new_int_value = self._transform(new_value) + + if self.lower is not None and self.upper is not None: + int_value = min(max(int_value, self.lower), self.upper) + new_int_value = min(max(new_int_value, self.lower), self.upper) + if int_value != new_int_value: rejected = False elif iteration > 100000: @@ -1332,9 +1459,9 @@ cdef class CategoricalHyperparameter(Hyperparameter): Additionally, it defines the __ne__() as stated in the documentation from python: - By default, object implements __eq__() by using is, returning NotImplemented - in the case of a false comparison: True if x is y else NotImplemented. - For __ne__(), by default it delegates to __eq__() and inverts the result + By default, object implements __eq__() by using is, returning NotImplemented + in the case of a false comparison: True if x is y else NotImplemented. + For __ne__(), by default it delegates to __eq__() and inverts the result unless it is NotImplemented. """ @@ -1588,9 +1715,9 @@ cdef class OrdinalHyperparameter(Hyperparameter): Additionally, it defines the __ne__() as stated in the documentation from python: - By default, object implements __eq__() by using is, returning NotImplemented - in the case of a false comparison: True if x is y else NotImplemented. - For __ne__(), by default it delegates to __eq__() and inverts the result + By default, object implements __eq__() by using is, returning NotImplemented + in the case of a false comparison: True if x is y else NotImplemented. + For __ne__(), by default it delegates to __eq__() and inverts the result unless it is NotImplemented. """ diff --git a/setup.py b/setup.py index 6acf864b..d477cabf 100644 --- a/setup.py +++ b/setup.py @@ -50,7 +50,7 @@ def finalize_options(self): AUTHOR_EMAIL = 'feurerm@informatik.uni-freiburg.de' TEST_SUITE = "pytest" SETUP_REQS = ['numpy', 'cython'] -INSTALL_REQS = ['numpy', 'cython', 'pyparsing'] +INSTALL_REQS = ['numpy', 'cython', 'pyparsing', 'scipy'] MIN_PYTHON_VERSION = '>=3.7' CLASSIFIERS = ['Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', diff --git a/test/test_hyperparameters.py b/test/test_hyperparameters.py index c453a096..ce5db573 100644 --- a/test/test_hyperparameters.py +++ b/test/test_hyperparameters.py @@ -194,6 +194,9 @@ def test_normalfloat(self): self.assertEqual( "param, Type: NormalFloat, Mu: 0.5 Sigma: 10.5, Default: 0.5", str(f1)) + self.assertEqual(f1.get_neighbors(0.5, rs=np.random.RandomState(42)), + [5.715498606617943, -0.9517751622974389, 7.300729650057271, + 16.491813492284265]) # Test attributes are accessible self.assertEqual(f1.name, "param") @@ -204,6 +207,14 @@ def test_normalfloat(self): self.assertAlmostEqual(f1.default_value, 0.5) self.assertAlmostEqual(f1.normalized_default_value, 0.5) + # Test copy + copy_f1 = copy.copy(f1) + + self.assertEqual(copy_f1.name, f1.name) + self.assertEqual(copy_f1.mu, f1.mu) + self.assertEqual(copy_f1.sigma, f1.sigma) + self.assertEqual(copy_f1.default_value, f1.default_value) + f2 = NormalFloatHyperparameter("param", 0, 10, q=0.1) f2_ = NormalFloatHyperparameter("param", 0, 10, q=0.1) self.assertEqual(f2, f2_) @@ -239,6 +250,32 @@ def test_normalfloat(self): self.assertNotEqual(f1, f2) self.assertNotEqual(f1, "UniformFloat") + with pytest.raises(ValueError): + f6 = NormalFloatHyperparameter("param", 5, 10, lower=0.1, upper=0.1, + default_value=5.0, q=0.1, log=True) + + with pytest.raises(ValueError): + f6 = NormalFloatHyperparameter("param", 5, 10, lower=0.1, default_value=5.0, + q=0.1, log=True) + + with pytest.raises(ValueError): + f6 = NormalFloatHyperparameter("param", 5, 10, upper=0.1, default_value=5.0, + q=0.1, log=True) + + f6 = NormalFloatHyperparameter("param", 5, 10, lower=0.1, upper=10, + default_value=5.0, q=0.1, log=True) + f6_ = NormalFloatHyperparameter("param", 5, 10, lower=0.1, upper=10, + default_value=5.0, q=0.1, log=True) + self.assertEqual(f6, f6_) + self.assertEqual( + "param, Type: NormalFloat, Mu: 5.0 Sigma: 10.0, Range: [0.1, 10.0], " + + "Default: 5.0, on log-scale, Q: 0.1", str(f6)) + self.assertEqual(f6.get_neighbors(5, rs=np.random.RandomState(42)), + [9.967141530112327, 3.6173569882881536, 10.0, 10.0]) + + self.assertNotEqual(f1, f2) + self.assertNotEqual(f1, "UniformFloat") + # test that meta-data is stored correctly f_meta = NormalFloatHyperparameter("param", 0.1, 10, q=0.1, log=True, default_value=1.0, meta=dict(self.meta_data)) @@ -250,6 +287,11 @@ def test_normalfloat_to_uniformfloat(self): f1_actual = f1.to_uniform() self.assertEqual(f1_expected, f1_actual) + f2 = NormalFloatHyperparameter("param", 0, 10, lower=-20, upper=20, q=0.1) + f2_expected = UniformFloatHyperparameter("param", -20, 20, q=0.1) + f2_actual = f2.to_uniform() + self.assertEqual(f2_expected, f2_actual) + def test_normalfloat_is_legal(self): f1 = NormalFloatHyperparameter("param", 0, 10) self.assertTrue(f1.is_legal(3.0)) @@ -627,6 +669,25 @@ def actual_test(): self.assertEqual(actual_test(), actual_test()) + def test_sample_NormalFloatHyperparameter_with_bounds(self): + hp = NormalFloatHyperparameter("nfhp", 0, 1, lower=-3, upper=3) + + def actual_test(): + rs = np.random.RandomState(1) + counts_per_bin = [0 for i in range(11)] + for i in range(100000): + value = hp.sample(rs) + index = min(max(int((np.round(value + 0.5)) + 5), 0), 9) + counts_per_bin[index] += 1 + + self.assertEqual([0, 0, 0, 2184, 13752, 34078, 34139, 13669, + 2178, 0, 0], counts_per_bin) + + self.assertIsInstance(value, float) + return counts_per_bin + + self.assertEqual(actual_test(), actual_test()) + def test_sample_UniformIntegerHyperparameter(self): # TODO: disentangle, actually test _sample and test sample on the # base class @@ -1118,6 +1179,11 @@ def test_hyperparam_representation(self): "param, Type: NormalFloat, Mu: 8.0 Sigma: 99.1, Default: 8.0", repr(f2) ) + f3 = NormalFloatHyperparameter("param", 8, 99.1, log=False, lower=1, upper=16) + self.assertEqual( + "param, Type: NormalFloat, Mu: 8.0 Sigma: 99.1, Range: [1.0, 16.0], Default: 8.0", + repr(f3) + ) i1 = UniformIntegerHyperparameter("param", 0, 100) self.assertEqual( "param, Type: UniformInteger, Range: [0, 100], Default: 50", @@ -1128,6 +1194,11 @@ def test_hyperparam_representation(self): "param, Type: NormalInteger, Mu: 5 Sigma: 8, Default: 5", repr(i2) ) + i3 = NormalIntegerHyperparameter("param", 5, 8, lower=1, upper=10) + self.assertEqual( + "param, Type: NormalInteger, Mu: 5 Sigma: 8, Range: [1, 10], Default: 5", + repr(i3) + ) o1 = OrdinalHyperparameter("temp", ["freezing", "cold", "warm", "hot"]) self.assertEqual( "temp, Type: Ordinal, Sequence: {freezing, cold, warm, hot}, Default: freezing",