add base implementation of optimum in OptimizerInterface

.gitignore

@@ -428,3 +428,6 @@ website/themes
 website/sphinx/_build
 website/sphinx/api
 website/python_api
+
+# python code coverage
+coverage

source/Mlos.Python/mlos/Grpc/BayesianOptimizerProxy.py

@@ -10,12 +10,12 @@
 from mlos.global_values import deserialize_from_bytes_string
 from mlos.Grpc import OptimizerService_pb2, OptimizerService_pb2_grpc
 from mlos.Logger import create_logger
-from mlos.Optimizers.OptimizerInterface import OptimizerInterface
+from mlos.Optimizers.OptimizerBase import OptimizerBase
 from mlos.Optimizers.RegressionModels.Prediction import Prediction
 from mlos.Spaces import Point
 
 
-class BayesianOptimizerProxy(OptimizerInterface):
+class BayesianOptimizerProxy(OptimizerBase):
  """ Client to remote BayesianOptimizer.
 
  Wraps all implementation details around communicating with the remote BayesianOptimizer.
@@ -37,7 +37,7 @@ def __init__(
  logger = create_logger("BayesianOptimizerClient")
  self.logger = logger
 
- OptimizerInterface.__init__(self, optimization_problem)
+ OptimizerBase.__init__(self, optimization_problem)
  assert optimizer_config is not None
 
  self._grpc_channel = grpc_channel
@@ -98,7 +98,7 @@ def predict(self, feature_values_pandas_frame, t=None): # pylint: disable=unuse
  )
  prediction_response = self._optimizer_stub.Predict(prediction_request)
 
- # To be compliant with the OptimizerInterface, we need to recover a single Prediction object and return it.
+ # To be compliant with the OptimizerBase, we need to recover a single Prediction object and return it.
  #
  objective_predictions_pb2 = prediction_response.ObjectivePredictions
  assert len(objective_predictions_pb2) == 1
@@ -109,9 +109,6 @@ def predict(self, feature_values_pandas_frame, t=None): # pylint: disable=unuse
  prediction.add_invalid_rows_at_missing_indices(desired_index=feature_values_pandas_frame.index)
  return prediction
 
- def optimum(self, stay_focused=False): # pylint: disable=unused-argument,no-self-use
- ...
-
  def focus(self, subspace): # pylint: disable=unused-argument,no-self-use
  ...
 

source/Mlos.Python/mlos/OptimizerMonitoring/Tomograph/ModelTomograph.py

@@ -6,7 +6,7 @@
 import numpy as np
 import pandas as pd
 
-from mlos.Optimizers.OptimizerInterface import OptimizerInterface
+from mlos.Optimizers.OptimizerBase import OptimizerBase
 from mlos.Optimizers.RegressionModels.Prediction import Prediction
 from mlos.Spaces import Point
 from mlos.OptimizerMonitoring.Tomograph.Heatmap import Heatmap
@@ -23,7 +23,7 @@ class ModelTomograph:
 
  def __init__(
  self,
- optimizer: OptimizerInterface,
+ optimizer: OptimizerBase,
  resolution: int = DEFAULT_RESOLUTION,
  dimension_names_to_skip=None, # TODO: remove - add an adapter that always removes useless dimension names.
  figure_size=(10, 10)
@@ -50,7 +50,7 @@ def __init__(
  is missing some important sectors of the search space.
 
 
- :param optimizer: a reference to an object implementing the OptimizerInterface.
+ :param optimizer: a reference to an object implementing the OptimizerBase.
  :param resolution: maximum number of pixels along a dimension of each heatmap.
  :param dimension_names_to_skip: dimensions not to be plotted. Remove this. Consider a solution, where mutually exclusive subgrids are plotted on
  separate figures.

source/Mlos.Python/mlos/Optimizers/BayesianOptimizer.py

@@ -9,7 +9,7 @@
 from mlos.Spaces import CategoricalDimension, DiscreteDimension, Point, SimpleHypergrid, DefaultConfigMeta
 
 from mlos.Optimizers.BayesianOptimizerConvergenceState import BayesianOptimizerConvergenceState
-from mlos.Optimizers.OptimizerInterface import OptimizerInterface
+from mlos.Optimizers.OptimizerBase import OptimizerBase
 from mlos.Optimizers.OptimizationProblem import OptimizationProblem
 from mlos.Optimizers.ExperimentDesigner.ExperimentDesigner import ExperimentDesigner, ExperimentDesignerConfig
 from mlos.Optimizers.RegressionModels.GoodnessOfFitMetrics import DataSetType
@@ -43,7 +43,7 @@ class BayesianOptimizerConfig(metaclass=DefaultConfigMeta):
  )
 
 
-class BayesianOptimizer(OptimizerInterface):
+class BayesianOptimizer(OptimizerBase):
  """Generic Bayesian Optimizer based on regresson model
 
  Uses extra trees as surrogate model and confidence bound acquisition function by default.
@@ -69,7 +69,7 @@ def __init__(
  # Let's initialize the optimizer.
  #
  assert len(optimization_problem.objectives) == 1, "For now this is a single-objective optimizer."
- OptimizerInterface.__init__(self, optimization_problem)
+ OptimizerBase.__init__(self, optimization_problem)
 
  assert optimizer_config in BayesianOptimizerConfig.CONFIG_SPACE, "Invalid config."
  self.optimizer_config = optimizer_config
@@ -142,27 +142,6 @@ def register(self, feature_values_pandas_frame, target_values_pandas_frame):
  def predict(self, feature_values_pandas_frame, t=None):
  return self.surrogate_model.predict(feature_values_pandas_frame)
 
- @trace()
- def optimum(self, stay_focused=False):
- if self.optimization_problem.objectives[0].minimize:
- index_of_best_target = self._target_values_df.idxmin()[0]
- else:
- index_of_best_target = self._target_values_df.idxmax()[0]
- objective_name = self.optimization_problem.objectives[0].name
- best_objective_value = self._target_values_df.loc[index_of_best_target][objective_name]
-
- param_names = [dimension.name for dimension in self.optimization_problem.parameter_space.dimensions]
- params_for_best_objective = self._feature_values_df.loc[index_of_best_target]
-
- optimal_config_and_target = {
- objective_name: best_objective_value,
- }
-
- for param_name in param_names:
- optimal_config_and_target[param_name] = params_for_best_objective[param_name]
-
- return optimal_config_and_target
-
  def focus(self, subspace):
  ...
 

source/Mlos.Python/mlos/Optimizers/OptimizerInterface.py → source/Mlos.Python/mlos/Optimizers/OptimizerBase.py

@@ -11,7 +11,7 @@
 from mlos.Optimizers.RegressionModels.Prediction import Prediction
 from mlos.Spaces import Point
 
-class OptimizerInterface(ABC):
+class OptimizerBase(ABC):
  """ Defines the interface to all our optimizers.
 
  """
@@ -56,15 +56,29 @@ def predict(self, feature_values_pandas_frame, t=None) -> Prediction:
  """
  raise NotImplementedError("All subclasses must implement this method.")
 
- @abstractmethod
- def optimum(self, stay_focused=False) -> Dict: # TODO: make it return an object
+ def optimum(self, stay_focused=False) -> Dict: # pylint: disable=unused-argument # TODO take context
  """ Return the optimal value found so far along with the related parameter values.
 
  This could be either min or max, depending on the settings.
 
- :return:
+ Returns
+ -------
+ best_config_point : Point
+ Configuration that corresponds to the optimum objective value.
+ best_objective : Point
+ Best objective value observed so far.
  """
- raise NotImplementedError("All subclasses must implement this method.")
+ features_df, objectives_df = self.get_all_observations()
+
+ if self.optimization_problem.objectives[0].minimize:
+ index_of_best_target = objectives_df.idxmin()[0]
+ else:
+ index_of_best_target = objectives_df.idxmax()[0]
+ best_objective = Point.from_dataframe(objectives_df.loc[[index_of_best_target]])
+ best_config_point = Point.from_dataframe(features_df.loc[[index_of_best_target]])
+
+
+ return best_config_point, best_objective
 
  @abstractmethod
  def focus(self, subspace):

source/Mlos.Python/mlos/Optimizers/SimpleBayesianOptimizer.py

@@ -11,7 +11,7 @@
 
 from mlos.Spaces import CategoricalDimension, ContinuousDimension, Dimension, DiscreteDimension, SimpleHypergrid, Point, DefaultConfigMeta
 from .OptimizationProblem import OptimizationProblem
-from .OptimizerInterface import OptimizerInterface
+from .OptimizerBase import OptimizerBase
 
 
 class SimpleBayesianOptimizerConfig(metaclass=DefaultConfigMeta):
@@ -76,14 +76,14 @@ def to_dict(self):
  }
 
 
-class SimpleBayesianOptimizer(OptimizerInterface):
+class SimpleBayesianOptimizer(OptimizerBase):
  """ A toy bayesian optimizer based on Gaussian processes.
 
  """
 
  def __init__(self, optimization_problem: OptimizationProblem, optimizer_config: SimpleBayesianOptimizerConfig):
  assert len(optimization_problem.objectives) == 1, "This is a single-objective optimizer."
- OptimizerInterface.__init__(self, optimization_problem)
+ OptimizerBase.__init__(self, optimization_problem)
  self.minimize = self.optimization_problem.objectives[0].minimize
 
  self._ordered_parameter_names = [
@@ -191,7 +191,7 @@ def suggest(self, random=False, context=None): # pylint: disable=redefined-oute
  return suggested_params
 
  def register(self, params, target_value): # pylint: disable=arguments-differ
- # TODO: make this conform to the OptimizerInterface
+ # TODO: make this conform to the OptimizerBase
 
  if params in self._registered_param_combos:
  return
@@ -300,18 +300,6 @@ def estimate_local_parameter_importance(self, params, t=None):
 
  return local_parameter_importance
 
- def optimum(self, stay_focused=False):
- # TODO: add arguments to set context
- self._optimizer._space._bounds = self._full_feature_space_bounds
-
- if stay_focused and self.focused:
- self._optimizer._space._bounds = self._format_parameter_bounds(self._focused_parameter_space_bounds)
-
- optimal_config_and_target = self._optimizer.max
- if self.minimize:
- optimal_config_and_target['target'] = -optimal_config_and_target['target']
- return optimal_config_and_target
-
  def focus(self, subspace):
  assert subspace in self.parameter_space
  parameter_bounds = self._format_search_space(subspace)

source/Mlos.Python/mlos/Optimizers/long_haul_tests/LongHaulTestBayesianOptimizer.py

@@ -115,8 +115,7 @@ def test_bayesian_optimizer_on_simple_2d_quadratic_function_pre_heated(self):
 
  # Register the observation with the optimizer
  bayesian_optimizer.register(input_values_df, target_values_df)
-
- self.logger.info(f"Optimum: {bayesian_optimizer.optimum()}")
+ self.logger.info(f"Optimum: {bayesian_optimizer.optimum()[1]}")
  trace_output_path = os.path.join(self.temp_dir, "PreHeatedTrace.json")
  self.logger.info(f"Writing trace to {trace_output_path}")
  global_values.tracer.dump_trace_to_file(output_file_path=trace_output_path)
@@ -167,9 +166,9 @@ def test_bayesian_optimizer_on_simple_2d_quadratic_function_cold_start(self):
 
  bayesian_optimizer.register(input_values_df, target_values_df)
  if i > 20 and i % 20 == 0:
- self.logger.info(f"[{i}/{num_guided_samples}] Optimum: {bayesian_optimizer.optimum()}")
+ self.logger.info(f"[{i}/{num_guided_samples}] Optimum: {bayesian_optimizer.optimum()[1]}")
 
- self.logger.info(f"Optimum: {bayesian_optimizer.optimum()}")
+ self.logger.info(f"Optimum: {bayesian_optimizer.optimum()[1]}")
 
  def test_hierarchical_quadratic_cold_start(self):
 
@@ -207,7 +206,7 @@ def test_hierarchical_quadratic_cold_start(self):
  target_values_df = pd.DataFrame({'y': [y]})
  bayesian_optimizer.register(input_values_df, target_values_df)
 
- self.logger.info(f"[{restart_num}/{num_restarts}] Optimum: {bayesian_optimizer.optimum()}")
+ self.logger.info(f"[{restart_num}/{num_restarts}] Optimum: {bayesian_optimizer.optimum()[1]}")
 
  def test_hierarchical_quadratic_cold_start_random_configs(self):
 
@@ -257,7 +256,7 @@ def test_hierarchical_quadratic_cold_start_random_configs(self):
  target_values_df = pd.DataFrame({'y': [y]})
  bayesian_optimizer.register(input_values_df, target_values_df)
 
- self.logger.info(f"[Restart: {restart_num}/{num_restarts}] Optimum: {bayesian_optimizer.optimum()}")
+ self.logger.info(f"[Restart: {restart_num}/{num_restarts}] Optimum: {bayesian_optimizer.optimum()[1]}")
  except Exception as e:
  has_failed = True
  error_file_path = os.path.join(os.getcwd(), "temp", "test_errors.txt")
@@ -322,5 +321,5 @@ def run_optimization(optimizer):
 
  for _ in range(40):
  run_optimization(optimizer)
- print(optimizer.optimum()['function_value'])
- self.assertLessEqual(sign * optimizer.optimum()['function_value'], -5.5)
+ print(optimizer.optimum()[1]['function_value'])
+ self.assertLessEqual(sign * optimizer.optimum()[1]['function_value'], -5.5)

source/Mlos.Python/mlos/Optimizers/unit_tests/TestBayesianOptimizer.py

@@ -121,7 +121,7 @@ def test_bayesian_optimizer_on_simple_2d_quadratic_function_pre_heated(self):
  # Register the observation with the optimizer
  bayesian_optimizer.register(input_values_df, target_values_df)
 
- print(bayesian_optimizer.optimum())
+ print(bayesian_optimizer.optimum()[1])
 
  def test_bayesian_optimizer_on_simple_2d_quadratic_function_cold_start(self):
  """ Tests the bayesian optimizer on a simple quadratic function with no prior data.
@@ -175,7 +175,7 @@ def test_bayesian_optimizer_on_simple_2d_quadratic_function_cold_start(self):
 
  bayesian_optimizer.register(input_values_df, target_values_df)
  if i > optimizer_config.min_samples_required_for_guided_design_of_experiments and i % 10 == 1:
- print(f"[{i}/{num_iterations}] Optimum: {bayesian_optimizer.optimum()}")
+ print(f"[{i}/{num_iterations}] Optimum: {bayesian_optimizer.optimum()[1]}")
  convergence_state = bayesian_optimizer.get_optimizer_convergence_state()
  random_forest_fit_state = convergence_state.surrogate_model_fit_state
  random_forest_gof_metrics = random_forest_fit_state.current_train_gof_metrics
@@ -245,7 +245,7 @@ def test_hierarchical_quadratic_cold_start(self):
  target_values_df = pd.DataFrame({'y': [y]})
  bayesian_optimizer.register(input_values_df, target_values_df)
 
- print(f"[{restart_num}/{num_restarts}] Optimum: {bayesian_optimizer.optimum()}")
+ print(f"[{restart_num}/{num_restarts}] Optimum: {bayesian_optimizer.optimum()[1]}")
 
 
  def test_hierarchical_quadratic_cold_start_random_configs(self):
@@ -296,7 +296,7 @@ def test_hierarchical_quadratic_cold_start_random_configs(self):
  target_values_df = pd.DataFrame({'y': [y]})
  bayesian_optimizer.register(input_values_df, target_values_df)
 
- print(f"[Restart: {restart_num}/{num_restarts}] Optimum: {bayesian_optimizer.optimum()}")
+ print(f"[Restart: {restart_num}/{num_restarts}] Optimum: {bayesian_optimizer.optimum()[1]}")
  except Exception as e:
  has_failed = True
  error_file_path = os.path.join(os.getcwd(), "temp", "test_errors.txt")

source/Mlos.Python/mlos/unit_tests/TestBayesianOptimizerGrpcClient.py

@@ -7,6 +7,7 @@
 import warnings
 
 import grpc
+import numpy as np
 import pandas as pd
 
 import mlos.global_values as global_values
@@ -169,6 +170,7 @@ def test_optimizer_with_named_config(self):
  def optimize_quadratic(self, optimizer, num_iterations):
  registered_features_df = None
  registered_objectives_df = None
+ old_optimum = np.inf
  for _ in range(num_iterations):
  params = optimizer.suggest()
  params_dict = params.to_dict()
@@ -177,6 +179,8 @@ def optimize_quadratic(self, optimizer, num_iterations):
  prediction = optimizer.predict(features_df)
  prediction_df = prediction.get_dataframe()
 
+
+
  y = quadratic(**params_dict)
  print(f"Params: {params}, Actual: {y}, Prediction: {str(prediction_df)}")
 
@@ -192,4 +196,10 @@ def optimize_quadratic(self, optimizer, num_iterations):
  registered_objectives_df = objectives_df
  else:
  registered_objectives_df = registered_objectives_df.append(objectives_df, ignore_index=True)
+
+ best_params, optimum = optimizer.optimum()
+ # ensure current optimum doesn't go up
+ assert optimum.y <= old_optimum
+ old_optimum = optimum.y
+ print(f"Best Params: {best_params}, Best Value: {optimum.y}")
  return registered_features_df, registered_objectives_df