Adding a new synthetic function (#217)

Adding EnvelopedWaves and MultiObjectiveEnvelopedWaves as two new synthetic functions with many useful properties.

These are sine waves enveloped by another function, either linear, quadratic or another sine wave.
An enveloped sine wave produces complexity for the optimizer that allows us evaluate its behavior on non-trivial problems.
Simultaneously, sine waves have the following advantages over polynomials:
1. They have well known optima - even when we envelop the function with another sine wave, as long as we keep their frequencies harmonic, we can know exactly where the optimum is.
2. They cannot be well approximated by a polynomial (Taylor expansion is accurate only locally).
3. For multi-objective problems, we can manipulate the phase shift of each objective to control the shape of the pareto frontier.

The way multi-objective enveloped waves work is that we pass the same parameters through 1 or more single-objective enveloped waves functions.
One useful property is that we not only know where the optima for individual functions are (maxima of sine are easy to find),
but we can also know and control the shape of the pareto frontier, by controlling the phase difference between the individual
objectives. For example: a phase difference of 0, means that that the objective functions are overlaid on top of each other
and their optima are exactly on top of each other, so the pareto frontier is a single, optimal point
Alternatively, the phase difference of quarter-period, introduces a trade-off between the objectives where
y0 = sin(x)
and
y1 = sin(x - math.pi / 2) = -cos(x)
which yields a pareto frontier in a shape of a quarter-cirle.
Yet another option is to use a phase difference of math.pi. This yields a trade-off between the objectives where:
y0 = sin(x)
and
y1 = sin(x - math.pi) = -sin(x) = -y0
which yields a pareto frontier where a gain in one objective results in an equal loss in the other objective, so the shape
of that frontier is a diagonal of a square.

source/Mlos.Python/mlos/OptimizerEvaluationTools/ObjectiveFunctionConfigStore.py

@@ -2,11 +2,14 @@
 # Copyright (c) Microsoft Corporation.
 # Licensed under the MIT License.
 #
+from mlos.OptimizerEvaluationTools.SyntheticFunctions.EnvelopedWaves import EnvelopedWaves, enveloped_waves_config_store
 from mlos.OptimizerEvaluationTools.SyntheticFunctions.Flower import Flower
 from mlos.OptimizerEvaluationTools.SyntheticFunctions.Hypersphere import Hypersphere
 from mlos.OptimizerEvaluationTools.SyntheticFunctions.HypersphereConfigStore import hypersphere_config_store
 from mlos.OptimizerEvaluationTools.SyntheticFunctions.MultiObjectiveNestedPolynomialObjective import MultiObjectiveNestedPolynomialObjective, \
  multi_objective_nested_polynomial_config_space
+from mlos.OptimizerEvaluationTools.SyntheticFunctions.MultiObjectiveEnvelopedWaves import MultiObjectiveEnvelopedWaves, \
+ multi_objective_enveloped_waves_config_store
 from mlos.OptimizerEvaluationTools.SyntheticFunctions.NestedPolynomialObjective import NestedPolynomialObjective, nested_polynomial_objective_config_space
 from mlos.OptimizerEvaluationTools.SyntheticFunctions.PolynomialObjective import PolynomialObjective
 from mlos.OptimizerEvaluationTools.SyntheticFunctions.ThreeLevelQuadratic import ThreeLevelQuadratic
@@ -18,12 +21,14 @@
  name="objective_function",
  dimensions=[
  CategoricalDimension(name="implementation", values=[
+ EnvelopedWaves.__name__,
  Flower.__name__,
  NestedPolynomialObjective.__name__,
  PolynomialObjective.__name__,
  ThreeLevelQuadratic.__name__,
  Hypersphere.__name__,
  MultiObjectiveNestedPolynomialObjective.__name__,
+ MultiObjectiveEnvelopedWaves.__name__,
  ])
  ]
  ).join(
@@ -38,6 +43,12 @@
  ).join(
  subgrid=multi_objective_nested_polynomial_config_space,
  on_external_dimension=CategoricalDimension(name="implementation", values=[MultiObjectiveNestedPolynomialObjective.__name__])
+ ).join(
+ subgrid=enveloped_waves_config_store.parameter_space,
+ on_external_dimension=CategoricalDimension(name="implementation", values=[EnvelopedWaves.__name__])
+ ).join(
+ subgrid=multi_objective_enveloped_waves_config_store.parameter_space,
+ on_external_dimension=CategoricalDimension(name="implementation", values=[MultiObjectiveEnvelopedWaves.__name__])
  ),
  default=Point(
  implementation=PolynomialObjective.__name__,

source/Mlos.Python/mlos/OptimizerEvaluationTools/SyntheticFunctions/EnvelopedWaves.py

@@ -0,0 +1,157 @@
+#
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+#
+import math
+import numpy as np
+import pandas as pd
+
+from mlos.OptimizerEvaluationTools.ObjectiveFunctionBase import ObjectiveFunctionBase
+from mlos.Spaces import CategoricalDimension, ContinuousDimension, DiscreteDimension, Point, SimpleHypergrid, Hypergrid
+from mlos.Spaces.Configs import ComponentConfigStore
+
+enveloped_waves_config_space = SimpleHypergrid(
+ name="enveloped_waves_config",
+ dimensions=[
+ DiscreteDimension(name="num_params", min=1, max=100),
+ ContinuousDimension(name="num_periods", min=1, max=100),
+ ContinuousDimension(name="vertical_shift", min=0, max=10),
+ ContinuousDimension(name="phase_shift", min=0, max=10000),
+ ContinuousDimension(name="period", min=0, max=10 * math.pi, include_min=False),
+ CategoricalDimension(name="envelope_type", values=["linear", "quadratic", "sine", "none"])
+ ]
+).join(
+ subgrid=SimpleHypergrid(
+ name="linear_envelope_config",
+ dimensions=[
+ ContinuousDimension(name="slope", min=-100, max=100)
+ ]
+ ),
+ on_external_dimension=CategoricalDimension(name="envelope_type", values=["linear"])
+).join(
+ subgrid=SimpleHypergrid(
+ name="quadratic_envelope_config",
+ dimensions=[
+ ContinuousDimension(name="a", min=-100, max=100),
+ ContinuousDimension(name="p", min=-100, max=100),
+ ]
+ ),
+ on_external_dimension=CategoricalDimension(name="envelope_type", values=["quadratic"])
+).join(
+ subgrid=SimpleHypergrid(
+ name="sine_envelope_config",
+ dimensions=[
+ ContinuousDimension(name="amplitude", min=0, max=10, include_min=False),
+ ContinuousDimension(name="phase_shift", min=0, max=2 * math.pi),
+ ContinuousDimension(name="period", min=0, max=100 * math.pi, include_min=False),
+ ]
+ ),
+ on_external_dimension=CategoricalDimension(name="envelope_type", values=["sine"])
+)
+
+enveloped_waves_config_store = ComponentConfigStore(
+ parameter_space=enveloped_waves_config_space,
+ default=Point(
+ num_params=3,
+ num_periods=1,
+ vertical_shift=0,
+ phase_shift=0,
+ period=2 * math.pi,
+ envelope_type="none"
+ ),
+ description="TODO"
+)
+
+class EnvelopedWaves(ObjectiveFunctionBase):
+ """Sum of sine waves enveloped by another function, either linear, quadratic or another sine wave.
+ An enveloped sine wave produces complexity for the optimizer that allows us evaluate its behavior on non-trivial problems.
+ Simultaneously, sine waves have the following advantages over polynomials:
+ 1. They have well known optima - even when we envelop the function with another sine wave, as long as we keep their frequencies
+ harmonic, we can know exactly where the optimum is.
+ 2. They cannot be well approximated by a polynomial (Taylor expansion is accurate only locally).
+ 3. For multi-objective problems, we can manipulate the phase shift of each objective to control the shape of the pareto frontier.
+ How the function works?
+ -----------------------
+ When creating the function we specify:
+ 1. Amplitute, vertical_shift, phase-shift and period of the sine wave.
+ 2. Envelope:
+ 1. Linear: slope (no need y_intercept as it's included in the vertical_shift)
+ 2. Quadratic: a, p, q
+ 3. Sine: again amplitude, phase shift, and period (no need to specify the vertical shift again.
+ The function takes the form:
+ y(x) = sum(
+ amplitude * sin((x_i - phase_shift) / period) * envelope(x_i) + vertical_shift
+ for x_i
+ in x
+ )
+ WHERE:
+ envelope(x_i) = envelope.slope * x_i + envelope.y_intercept
+ OR
+ envelope(x_i) = a * (x_i - p) ** 2 + q
+ OR
+ envelope(x_i) = envelope.amplitude * sin((x_i - envelope.phase_shift) / envelope.period)
+ """
+
+ def __init__(self, objective_function_config: Point = None):
+ assert objective_function_config in enveloped_waves_config_space, f"{objective_function_config} not in {enveloped_waves_config_space}"
+ ObjectiveFunctionBase.__init__(self, objective_function_config)
+ self._parameter_space = SimpleHypergrid(
+ name="domain",
+ dimensions=[
+ ContinuousDimension(name=f"x_{i}", min=0, max=objective_function_config.num_periods * objective_function_config.period)
+ for i in range(self.objective_function_config.num_params)
+ ]
+ )
+
+ self._output_space = SimpleHypergrid(
+ name="range",
+ dimensions=[
+ ContinuousDimension(name="y", min=-math.inf, max=math.inf)
+ ]
+ )
+
+ if self.objective_function_config.envelope_type == "linear":
+ self._envelope = self._linear_envelope
+ elif self.objective_function_config.envelope_type == "quadratic":
+ self._envelope = self._quadratic_envelope
+ elif self.objective_function_config.envelope_type == "sine":
+ self._envelope = self._sine_envelope
+ else:
+ self._envelope = lambda x: x * 0 + 1
+
+ @property
+ def parameter_space(self) -> Hypergrid:
+ return self._parameter_space
+
+ @property
+ def output_space(self) -> Hypergrid:
+ return self._output_space
+
+
+ def evaluate_dataframe(self, dataframe: pd.DataFrame) -> pd.DataFrame:
+ objectives_df = pd.DataFrame(0, index=dataframe.index, columns=['y'], dtype='float')
+ for param_name in self._parameter_space.dimension_names:
+ objectives_df['y'] += np.sin(
+ dataframe[param_name] / self.objective_function_config.period * 2 * math.pi - self.objective_function_config.phase_shift
+ ) * self._envelope(dataframe[param_name])
+ objectives_df['y'] += self.objective_function_config.vertical_shift
+
+ return objectives_df
+
+ def _linear_envelope(self, x: pd.Series):
+ return x * self.objective_function_config.linear_envelope_config.slope
+
+ def _quadratic_envelope(self, x: pd.Series):
+ a = self.objective_function_config.quadratic_envelope_config.a
+ p = self.objective_function_config.quadratic_envelope_config.p
+ return a * (x - p) ** 2
+
+ def _sine_envelope(self, x: pd.Series):
+ amplitude = self.objective_function_config.sine_envelope_config.amplitude
+ phase_shift = self.objective_function_config.sine_envelope_config.phase_shift
+ period = self.objective_function_config.sine_envelope_config.period
+ return amplitude * np.sin(x / period - phase_shift)
+
+
+ def get_context(self) -> Point:
+ return self.objective_function_config

source/Mlos.Python/mlos/OptimizerEvaluationTools/SyntheticFunctions/MultiObjectiveEnvelopedWaves.py

@@ -0,0 +1,144 @@
+#
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT License.
+#
+import math
+import pandas as pd
+
+from mlos.OptimizerEvaluationTools.ObjectiveFunctionBase import ObjectiveFunctionBase
+from mlos.OptimizerEvaluationTools.SyntheticFunctions.EnvelopedWaves import EnvelopedWaves, enveloped_waves_config_store
+from mlos.Spaces import CategoricalDimension, ContinuousDimension, DiscreteDimension, Point, SimpleHypergrid, Hypergrid
+from mlos.Spaces.Configs import ComponentConfigStore
+from mlos.Utils.KeyOrderedDict import KeyOrderedDict
+
+
+multi_objective_enveloped_waves_config_space = SimpleHypergrid(
+ name="multi_objective_enveloped_waves_config",
+ dimensions=[
+ DiscreteDimension(name="num_objectives", min=1, max=10),
+ ContinuousDimension(name="phase_difference", min=0, max=2 * math.pi),
+ ContinuousDimension(name="period_change", min=1, max=1.2),
+ CategoricalDimension(name="single_objective_function", values=[EnvelopedWaves.__name__])
+ ]
+).join(
+ on_external_dimension=CategoricalDimension(name="single_objective_function", values=[EnvelopedWaves.__name__]),
+ subgrid=enveloped_waves_config_store.parameter_space
+)
+
+multi_objective_enveloped_waves_config_store = ComponentConfigStore(
+ parameter_space=multi_objective_enveloped_waves_config_space,
+ default=Point(
+ num_objectives=2,
+ phase_difference=0.5 * math.pi,
+ period_change=1.0,
+ single_objective_function=EnvelopedWaves.__name__,
+ enveloped_waves_config=enveloped_waves_config_store.default
+ ),
+ description="TODO"
+)
+
+multi_objective_enveloped_waves_config_store.add_config_by_name(
+ config_name="no_phase_difference",
+ config_point=Point(
+ num_objectives=2,
+ phase_difference=0,
+ period_change=1.0,
+ single_objective_function=EnvelopedWaves.__name__,
+ enveloped_waves_config=enveloped_waves_config_store.default
+ ),
+ description="This function should produce a pareto frontier consisting of a single point at all parameter values equal to (math.pi / 2)."
+)
+
+multi_objective_enveloped_waves_config_store.add_config_by_name(
+ config_name="half_pi_phase_difference",
+ config_point=Point(
+ num_objectives=2,
+ phase_difference=math.pi / 2,
+ period_change=1.0,
+ single_objective_function=EnvelopedWaves.__name__,
+ enveloped_waves_config=enveloped_waves_config_store.default
+ ),
+ description="This function should produce a pareto frontier consisting of points in a quarter cricle in a first quadrat with the radius equal to 3."
+)
+
+multi_objective_enveloped_waves_config_store.add_config_by_name(
+ config_name="pi_phase_difference",
+ config_point=Point(
+ num_objectives=2,
+ phase_difference=math.pi,
+ period_change=1.0,
+ single_objective_function=EnvelopedWaves.__name__,
+ enveloped_waves_config=enveloped_waves_config_store.default
+ ),
+ description="This function should produce a pareto frontier consisting of points on a diagonal of a square centered on the origin"
+ " with side length equal to 18."
+)
+
+class MultiObjectiveEnvelopedWaves(ObjectiveFunctionBase):
+ """Multi-objective function with many useful properties.
+ The way it works is that we pass the same parameters through 1 or more single-objective enveloped waves functions.
+ One useful property is that we not only know where the optima for individual functions are (maxima of sine are easy to find),
+ but we can also know and control the shape of the pareto frontier, by controlling the phase difference between the individual
+ objectives. For example: a phase difference of 0, means that that the objective functions are overlaid on top of each other
+ and their optima are exactly on top of each other, so the pareto frontier is a single, optimal point
+ Alternatively, the phase difference of quarter-period, introduces a trade-off between the objectives where
+ y0 = sin(x)
+ and
+ y1 = sin(x - math.pi / 2) = -cos(x)
+ which yields a pareto frontier in a shape of a quarter-cirle of radius 1 (or amplitude more generally).
+ Yet another option is to use a phase difference of math.pi. This yields a trade-off between the objectives where:
+ y0 = sin(x)
+ and
+ y1 = sin(x - math.pi) = -sin(x) = -y0
+ which yields a pareto frontier where a gain in one objective results in an equal loss in the other objective, so the shape
+ of that frontier is a diagonal of a square with side length equal to amplitude.
+ """
+
+ def __init__(self, objective_function_config: Point = None):
+ assert objective_function_config in multi_objective_enveloped_waves_config_space
+ ObjectiveFunctionBase.__init__(self, objective_function_config)
+ single_objective_enveloped_waves_config = objective_function_config.enveloped_waves_config
+ self._individual_objectives = KeyOrderedDict(
+ ordered_keys=[f"y{objective_id}" for objective_id in range(objective_function_config.num_objectives)],
+ value_type=EnvelopedWaves
+ )
+
+ for objective_id in range(objective_function_config.num_objectives):
+ config = single_objective_enveloped_waves_config.copy()
+ config.phase_shift += objective_function_config.phase_difference * objective_id
+ config.period *= objective_function_config.period_change ** objective_id
+
+ while config.period > enveloped_waves_config_store.parameter_space["period"].max:
+ config.period -= enveloped_waves_config_store.parameter_space["period"].max
+
+ while config.phase_shift > enveloped_waves_config_store.parameter_space["phase_shift"].max:
+ config.phase_shift -= enveloped_waves_config_store.parameter_space["phase_shift"].max
+
+ self._individual_objectives[objective_id] = EnvelopedWaves(objective_function_config=config)
+
+ self._parameter_space = self._individual_objectives[0].parameter_space
+ self._output_space = SimpleHypergrid(
+ name="range",
+ dimensions=[
+ ContinuousDimension(name=f"y{objective_id}", min=-math.inf, max=math.inf)
+ for objective_id in range(objective_function_config.num_objectives)
+ ]
+ )
+
+ @property
+ def parameter_space(self) -> Hypergrid:
+ return self._parameter_space
+
+ @property
+ def output_space(self) -> Hypergrid:
+ return self._output_space
+
+ def evaluate_dataframe(self, dataframe: pd.DataFrame) -> pd.DataFrame:
+ results_df = pd.DataFrame()
+ for objective_dim_name, individual_objective_function in self._individual_objectives:
+ single_objective_df = individual_objective_function.evaluate_dataframe(dataframe)
+ results_df[objective_dim_name] = single_objective_df['y']
+ return results_df
+
+ def get_context(self) -> Point:
+ return self.objective_function_config

source/Mlos.Python/mlos/OptimizerEvaluationTools/SyntheticFunctions/PolynomialObjectiveWrapper.py

@@ -61,10 +61,12 @@ def output_space(self) -> Hypergrid:
  return self._output_space
 
  def evaluate_point(self, point: Point) -> Point:
+ point = Point(**{dim_name: point[dim_name] for dim_name in self._parameter_space.dimension_names})
  y = self._polynomial_function.evaluate(point.to_dataframe().to_numpy())
  return Point(y=y[0])
 
  def evaluate_dataframe(self, dataframe: pd.DataFrame) -> pd.DataFrame:
+ dataframe = dataframe[self._parameter_space.dimension_names]
  y = self._polynomial_function.evaluate(dataframe.to_numpy())
  return pd.DataFrame({'y': y})