Suggest int parameter with step

_benchmarks/optuna_solver/cmaes.py

@@ -27,7 +27,7 @@ def specification(self):
 
 
 def create_study(seed):
-    sampler = optuna.integration.CmaEsSampler(
+    sampler = optuna.samplers.CmaEsSampler(
         seed=seed,
         n_startup_trials=args.startup,
         warn_independent_sampling=True,

_benchmarks/optuna_solver/requirements.txt

@@ -1,3 +1,2 @@
 optuna
-cma
 kurobako

_benchmarks/run_himmelblau.sh

@@ -12,11 +12,12 @@ RANDOM_SOLVER=$($KUROBAKO solver random)
 
 CMA_SOLVER=$($KUROBAKO solver command ./cma_solver)
 TPE_SOLVER=$($KUROBAKO solver command ./tpe_solver)
-OPTUNA_SOLVER=$($KUROBAKO solver command python ./_benchmarks/optuna_solver/cmaes.py)
+# It has a circular import problem.
+# OPTUNA_SOLVER=$($KUROBAKO solver command python ./_benchmarks/optuna_solver/cmaes.py)
 PROBLEM=$($KUROBAKO problem command ./himmelblau_problem)
 
 $KUROBAKO studies \
-  --solvers $RANDOM_SOLVER $TPE_SOLVER $CMA_SOLVER $OPTUNA_SOLVER \
+  --solvers $RANDOM_SOLVER $TPE_SOLVER $CMA_SOLVER \
   --problems $PROBLEM \
   --repeats 8 --budget 300 \
   | $KUROBAKO run --parallelism 1 > $1

_benchmarks/run_rosenbrock.sh

@@ -12,11 +12,12 @@ RANDOM_SOLVER=$($KUROBAKO solver random)
 
 CMA_SOLVER=$($KUROBAKO solver command ./cma_solver)
 TPE_SOLVER=$($KUROBAKO solver command ./tpe_solver)
-OPTUNA_SOLVER=$($KUROBAKO solver command python ./_benchmarks/optuna_solver/cmaes.py)
+# It has a circular import problem.
+# OPTUNA_SOLVER=$($KUROBAKO solver command python ./_benchmarks/optuna_solver/cmaes.py)
 PROBLEM=$($KUROBAKO problem command ./rosenbrock_problem)
 
 $KUROBAKO studies \
-  --solvers $RANDOM_SOLVER $TPE_SOLVER $CMA_SOLVER $OPTUNA_SOLVER \
+  --solvers $RANDOM_SOLVER $TPE_SOLVER $CMA_SOLVER \
   --problems $PROBLEM \
   --repeats 8 --budget 300 \
   | $KUROBAKO run --parallelism 1 > $1

cmaes/sampler.go

@@ -86,13 +86,7 @@ func (s *Sampler) SampleRelative(
 			if !ok {
 				return nil, errors.New("invalid internal params")
 			}
-
-			switch searchSpace[orderedKeys[j]].(type) {
-			case goptuna.LogUniformDistribution:
-				p = math.Log(p)
-			}
-
-			x[j] = p
+			x[j] = toCMAParam(searchSpace[orderedKeys[j]], p)
 		}
 		solutions = append(solutions, &Solution{
 			Params: x,
@@ -124,11 +118,7 @@ func (s *Sampler) SampleRelative(
 	params := make(map[string]float64, len(orderedKeys))
 	for i := range orderedKeys {
 		param := nextParams[i]
-		switch searchSpace[orderedKeys[i]].(type) {
-		case goptuna.LogUniformDistribution:
-			param = math.Exp(param)
-		}
-		params[orderedKeys[i]] = param
+		params[orderedKeys[i]] = toGoptunaInternalParam(searchSpace[orderedKeys[i]], param)
 	}
 	return params, nil
 }
@@ -192,11 +182,29 @@ func supportedSearchSpace(searchSpace map[string]interface{}) map[string]interfa
 			normalized[name] = searchSpace[name]
 		case goptuna.IntUniformDistribution:
 			normalized[name] = searchSpace[name]
+		case goptuna.StepIntUniformDistribution:
+			normalized[name] = searchSpace[name]
 		}
 	}
 	return normalized
 }
 
+func toCMAParam(distribution interface{}, goptunaParam float64) float64 {
+	switch distribution.(type) {
+	case goptuna.LogUniformDistribution:
+		return math.Log(goptunaParam)
+	}
+	return goptunaParam
+}
+
+func toGoptunaInternalParam(distribution interface{}, cmaParam float64) float64 {
+	switch distribution.(type) {
+	case goptuna.LogUniformDistribution:
+		return math.Exp(cmaParam)
+	}
+	return cmaParam
+}
+
 func initialParam(searchSpace map[string]interface{}) (map[string]float64, float64, error) {
 	x0 := make(map[string]float64, len(searchSpace))
 	sigma0 := make([]float64, 0, len(searchSpace))
@@ -216,6 +224,9 @@ func initialParam(searchSpace map[string]interface{}) (map[string]float64, float
 		case goptuna.IntUniformDistribution:
 			x0[name] = float64(d.High+d.Low) / 2
 			sigma0 = append(sigma0, float64(d.High-d.Low)/6)
+		case goptuna.StepIntUniformDistribution:
+			x0[name] = float64(d.High+d.Low) / 2
+			sigma0 = append(sigma0, float64(d.High-d.Low)/6)
 		default:
 			return nil, 0, goptuna.ErrUnknownDistribution
 		}
@@ -242,6 +253,9 @@ func getSearchSpaceBounds(
 		case goptuna.IntUniformDistribution:
 			bounds.Set(i, 0, float64(d.Low))
 			bounds.Set(i, 1, float64(d.High))
+		case goptuna.StepIntUniformDistribution:
+			bounds.Set(i, 0, float64(d.Low))
+			bounds.Set(i, 1, float64(d.High))
 		default:
 			panic("keys and search_space do not match")
 		}

distribution.go

@@ -109,6 +109,43 @@ func (d *IntUniformDistribution) Contains(ir float64) bool {
 	return d.Low <= value && value < d.High
 }
 
+// StepIntUniformDistributionName is the identifier name of IntUniformDistribution
+const StepIntUniformDistributionName = "StepIntUniformDistribution"
+
+// StepIntUniformDistribution is a uniform distribution on integers.
+type StepIntUniformDistribution struct {
+	// High is higher endpoint of the range of the distribution (included in the range).
+	High int `json:"high"`
+	// Low is lower endpoint of the range of the distribution (included in the range).
+	Low int `json:"low"`
+	// Step is a spacing between values.
+	Step int `json:"step"`
+}
+
+// ToExternalRepr to convert internal representation of a parameter value into external representation.
+func (d *StepIntUniformDistribution) ToExternalRepr(ir float64) interface{} {
+	r := (ir - float64(d.Low)) / float64(d.Step) // shift to [0, (high-low)/step]
+	v := int(math.Round(r))*d.Step + d.Low
+	return v
+}
+
+// Single to test whether the range of this distribution contains just a single value.
+func (d *StepIntUniformDistribution) Single() bool {
+	if d.High == d.Low {
+		return true
+	}
+	return (d.High - d.Low) < d.Step
+}
+
+// Contains to check a parameter value is contained in the range of this distribution.
+func (d *StepIntUniformDistribution) Contains(ir float64) bool {
+	value := int(ir)
+	if d.Single() {
+		return value == d.Low
+	}
+	return d.Low <= value && value < d.High
+}
+
 var _ Distribution = &DiscreteUniformDistribution{}
 
 // DiscreteUniformDistribution is a discretized uniform distribution in the linear domain.
@@ -189,6 +226,8 @@ func ToExternalRepresentation(distribution interface{}, ir float64) (interface{}
 		return d.ToExternalRepr(ir), nil
 	case IntUniformDistribution:
 		return d.ToExternalRepr(ir), nil
+	case StepIntUniformDistribution:
+		return d.ToExternalRepr(ir), nil
 	case DiscreteUniformDistribution:
 		return d.ToExternalRepr(ir), nil
 	case CategoricalDistribution:
@@ -207,6 +246,8 @@ func DistributionIsSingle(distribution interface{}) (bool, error) {
 		return d.Single(), nil
 	case IntUniformDistribution:
 		return d.Single(), nil
+	case StepIntUniformDistribution:
+		return d.Single(), nil
 	case DiscreteUniformDistribution:
 		return d.Single(), nil
 	case CategoricalDistribution:
@@ -231,6 +272,8 @@ func DistributionToJSON(distribution interface{}) ([]byte, error) {
 		ir.Name = IntUniformDistributionName
 	case DiscreteUniformDistribution:
 		ir.Name = DiscreteUniformDistributionName
+	case StepIntUniformDistribution:
+		ir.Name = StepIntUniformDistributionName
 	case CategoricalDistribution:
 		ir.Name = CategoricalDistributionName
 	default:
@@ -287,6 +330,15 @@ func JSONToDistribution(jsonBytes []byte) (interface{}, error) {
 		}
 		err = json.Unmarshal(dbytes, &y)
 		return y, err
+	case StepIntUniformDistributionName:
+		var y StepIntUniformDistribution
+		var dbytes []byte
+		dbytes, err = json.Marshal(x.Attrs)
+		if err != nil {
+			return nil, err
+		}
+		err = json.Unmarshal(dbytes, &y)
+		return y, err
 	case CategoricalDistributionName:
 		var y CategoricalDistribution
 		var dbytes []byte

distribution_test.go

@@ -41,6 +41,14 @@ func TestDistributionConversionBetweenDistributionAndJSON(t *testing.T) {
 				Q:    1,
 			},
 		},
+		{
+			name: "step int uniform distribution",
+			distribution: goptuna.StepIntUniformDistribution{
+				High: 20,
+				Low:  5,
+				Step: 2,
+			},
+		},
 		{
 			name: "categorical distribution",
 			distribution: goptuna.CategoricalDistribution{
@@ -90,6 +98,24 @@ func TestDistributionToExternalRepresentation(t *testing.T) {
 			args:         3.0,
 			want:         3,
 		},
+		{
+			name:         "step int uniform distribution 1",
+			distribution: &goptuna.StepIntUniformDistribution{Low: 0, High: 10, Step: 2},
+			args:         2.2,
+			want:         2,
+		},
+		{
+			name:         "step int uniform distribution 2",
+			distribution: &goptuna.StepIntUniformDistribution{Low: 0, High: 10, Step: 3},
+			args:         2.2,
+			want:         3,
+		},
+		{
+			name:         "step int uniform distribution 2",
+			distribution: &goptuna.StepIntUniformDistribution{Low: -1, High: 10, Step: 3},
+			args:         0.6,
+			want:         2,
+		},
 		{
 			name:         "discrete uniform distribution 1",
 			distribution: &goptuna.DiscreteUniformDistribution{Low: 0.5, High: 5.5, Q: 0.5},
@@ -170,6 +196,21 @@ func TestDistributionSingle(t *testing.T) {
 			distribution: &goptuna.DiscreteUniformDistribution{Low: 0, High: 10, Q: 1},
 			want:         false,
 		},
+		{
+			name:         "step int uniform distribution true",
+			distribution: &goptuna.StepIntUniformDistribution{Low: 10, High: 10, Step: 2},
+			want:         true,
+		},
+		{
+			name:         "step int uniform distribution true when step is greater than high-low",
+			distribution: &goptuna.StepIntUniformDistribution{Low: 0, High: 1, Step: 2},
+			want:         true,
+		},
+		{
+			name:         "step int uniform distribution false",
+			distribution: &goptuna.StepIntUniformDistribution{Low: 0, High: 10, Step: 2},
+			want:         false,
+		},
 		{
 			name:         "categorical distribution true",
 			distribution: &goptuna.CategoricalDistribution{Choices: []string{"a"}},
@@ -251,6 +292,24 @@ func TestDistributionContains(t *testing.T) {
 			args:         15,
 			want:         false,
 		},
+		{
+			name:         "step int uniform distribution true",
+			distribution: &goptuna.StepIntUniformDistribution{Low: 0, High: 10, Step: 2},
+			args:         3,
+			want:         true,
+		},
+		{
+			name:         "step int uniform distribution lower",
+			distribution: &goptuna.StepIntUniformDistribution{Low: 0, High: 10, Step: 2},
+			args:         -3,
+			want:         false,
+		},
+		{
+			name:         "step int uniform distribution higher",
+			distribution: &goptuna.StepIntUniformDistribution{Low: 0, High: 10, Step: 2},
+			args:         15,
+			want:         false,
+		},
 		{
 			name:         "discrete uniform distribution true",
 			distribution: &goptuna.DiscreteUniformDistribution{Low: 0.5, High: 5.5, Q: 1},

sampler.go

@@ -138,6 +138,13 @@ func (s *RandomSearchSampler) Sample(
 			return float64(d.Low), nil
 		}
 		return float64(s.rng.Intn(d.High-d.Low) + d.Low), nil
+	case StepIntUniformDistribution:
+		if d.Single() {
+			return float64(d.Low), nil
+		}
+		r := (d.High - d.Low) / d.Step
+		v := (s.rng.Intn(r) * d.Step) + d.Low
+		return float64(v), nil
 	case DiscreteUniformDistribution:
 		if d.Single() {
 			return d.Low, nil

structs.go

@@ -93,6 +93,10 @@ func (t FrozenTrial) validate() error {
 			if !typedDistribution.Contains(ir) {
 				return fmt.Errorf("internal param is out of the distribution range")
 			}
+		case StepIntUniformDistribution:
+			if !typedDistribution.Contains(ir) {
+				return fmt.Errorf("internal param is out of the distribution range")
+			}
 		case DiscreteUniformDistribution:
 			if !typedDistribution.Contains(ir) {
 				return fmt.Errorf("internal param is out of the distribution range")

tpe/sampler.go

@@ -402,6 +402,13 @@ func (s *Sampler) sampleInt(distribution goptuna.IntUniformDistribution, below,
 	return s.sampleNumerical(low, high, below, above, q, false)
 }
 
+func (s *Sampler) sampleStepInt(distribution goptuna.StepIntUniformDistribution, below, above []float64) float64 {
+	q := 1.0
+	low := float64(distribution.Low) - 0.5*q
+	high := float64(distribution.High) + 0.5*q
+	return s.sampleNumerical(low, high, below, above, q, false)
+}
+
 func (s *Sampler) sampleDiscreteUniform(distribution goptuna.DiscreteUniformDistribution, below, above []float64) float64 {
 	q := distribution.Q
 	r := distribution.High - distribution.Low
@@ -501,6 +508,8 @@ func (s *Sampler) Sample(
 		return s.sampleLogUniform(d, belowParamValues, aboveParamValues), nil
 	case goptuna.IntUniformDistribution:
 		return s.sampleInt(d, belowParamValues, aboveParamValues), nil
+	case goptuna.StepIntUniformDistribution:
+		return s.sampleStepInt(d, belowParamValues, aboveParamValues), nil
 	case goptuna.DiscreteUniformDistribution:
 		return s.sampleDiscreteUniform(d, belowParamValues, aboveParamValues), nil
 	case goptuna.CategoricalDistribution:

trial.go

@@ -83,6 +83,10 @@ func (t *Trial) isFixedParam(name string, distribution interface{}) (float64, bo
 		if !typedDistribution.Contains(internalParam) {
 			return 0, false, nil
 		}
+	case StepIntUniformDistribution:
+		if !typedDistribution.Contains(internalParam) {
+			return 0, false, nil
+		}
 	case CategoricalDistribution:
 		if !typedDistribution.Contains(internalParam) {
 			return 0, false, nil
@@ -188,10 +192,26 @@ func (t *Trial) SuggestInt(name string, low, high int) (int, error) {
 	if low > high {
 		return 0, errors.New("'low' must be smaller than or equal to the 'high'")
 	}
-	v, err := t.suggest(name, IntUniformDistribution{
+	d := IntUniformDistribution{
 		High: high, Low: low,
-	})
-	return int(v), err
+	}
+	v, err := t.suggest(name, d)
+	return d.ToExternalRepr(v).(int), err
+}
+
+// SuggestIntWithStep suggests an integer parameter.
+func (t *Trial) SuggestIntWithStep(name string, low, high, step int) (int, error) {
+	if low > high {
+		return 0, errors.New("'low' must be smaller than or equal to the 'high'")
+	}
+	if step <= 0 {
+		return 0, errors.New("'step' must be larger than 0")
+	}
+	d := StepIntUniformDistribution{
+		High: high, Low: low, Step: step,
+	}
+	v, err := t.suggest(name, d)
+	return d.ToExternalRepr(v).(int), err
 }
 
 // SuggestDiscreteUniform suggests a value from a discrete uniform distribution.