Require explicit arguments for mlos_core optimizers

mlos_bench/mlos_bench/optimizers/mlos_core_optimizer.py

@@ -117,7 +117,7 @@ def bulk_register(self,
 
  # TODO: Specify (in the config) which metrics to pass to the optimizer.
  # Issue: https://github.com/microsoft/MLOS/issues/745
- self._opt.register(df_configs, df_scores[opt_targets].astype(float))
+ self._opt.register(configurations=df_configs, scores=df_scores[opt_targets].astype(float))
 
  if _LOG.isEnabledFor(logging.DEBUG):
  (score, _) = self.get_best_observation()
@@ -195,7 +195,7 @@ def register(self, tunables: TunableGroups, status: Status,
  _LOG.debug("Score: %s Dataframe:\n%s", registered_score, df_config)
  # TODO: Specify (in the config) which metrics to pass to the optimizer.
  # Issue: https://github.com/microsoft/MLOS/issues/745
- self._opt.register(df_config, pd.DataFrame([registered_score], dtype=float))
+ self._opt.register(configurations=df_config, scores=pd.DataFrame([registered_score], dtype=float))
  return registered_score
 
  def get_best_observation(self) -> Union[Tuple[Dict[str, float], TunableGroups], Tuple[None, None]]:

mlos_bench/mlos_bench/tests/optimizers/toy_optimization_loop_test.py

@@ -50,7 +50,7 @@ def _optimize(env: Environment, opt: Optimizer) -> Tuple[float, TunableGroups]:
  config_df = config_to_dataframe(config)
  logger("config: %s", str(config))
  try:
- logger("prediction: %s", opt._opt.surrogate_predict(config_df))
+ logger("prediction: %s", opt._opt.surrogate_predict(configurations=config_df))
  except RuntimeError:
  pass
 

mlos_core/mlos_core/optimizers/bayesian_optimizers/bayesian_optimizer.py

@@ -20,7 +20,7 @@ class BaseBayesianOptimizer(BaseOptimizer, metaclass=ABCMeta):
  """Abstract base class defining the interface for Bayesian optimization."""
 
  @abstractmethod
- def surrogate_predict(self, configurations: pd.DataFrame,
+ def surrogate_predict(self, *, configurations: pd.DataFrame,
  context: Optional[pd.DataFrame] = None) -> npt.NDArray:
  """Obtain a prediction from this Bayesian optimizer's surrogate model for the given configuration(s).
 
@@ -35,7 +35,7 @@ def surrogate_predict(self, configurations: pd.DataFrame,
  pass # pylint: disable=unnecessary-pass # pragma: no cover
 
  @abstractmethod
- def acquisition_function(self, configurations: pd.DataFrame,
+ def acquisition_function(self, *, configurations: pd.DataFrame,
  context: Optional[pd.DataFrame] = None) -> npt.NDArray:
  """Invokes the acquisition function from this Bayesian optimizer for the given configuration.
 

mlos_core/mlos_core/optimizers/bayesian_optimizers/smac_optimizer.py

@@ -241,7 +241,7 @@ def _dummy_target_func(config: ConfigSpace.Configuration, seed: int = 0) -> None
  # -- this planned to be fixed in some future release: https://github.com/automl/SMAC3/issues/946
  raise RuntimeError('This function should never be called.')
 
- def _register(self, configurations: pd.DataFrame,
+ def _register(self, *, configurations: pd.DataFrame,
  scores: pd.DataFrame, context: Optional[pd.DataFrame] = None) -> None:
  """Registers the given configurations and scores.
 
@@ -262,7 +262,7 @@ def _register(self, configurations: pd.DataFrame,
  warn(f"Not Implemented: Ignoring context {list(context.columns)}", UserWarning)
 
  # Register each trial (one-by-one)
- for (config, (_i, score)) in zip(self._to_configspace_configs(configurations), scores.iterrows()):
+ for (config, (_i, score)) in zip(self._to_configspace_configs(configurations=configurations), scores.iterrows()):
  # Retrieve previously generated TrialInfo (returned by .ask()) or create new TrialInfo instance
  info: TrialInfo = self.trial_info_map.get(
  config, TrialInfo(config=config, seed=self.base_optimizer.scenario.seed))
@@ -272,7 +272,7 @@ def _register(self, configurations: pd.DataFrame,
  # Save optimizer once we register all configs
  self.base_optimizer.optimizer.save()
 
- def _suggest(self, context: Optional[pd.DataFrame] = None) -> pd.DataFrame:
+ def _suggest(self, *, context: Optional[pd.DataFrame] = None) -> pd.DataFrame:
  """Suggests a new configuration.
 
  Parameters
@@ -299,10 +299,10 @@ def _suggest(self, context: Optional[pd.DataFrame] = None) -> pd.DataFrame:
  config_df = pd.DataFrame([trial.config], columns=list(self.optimizer_parameter_space.keys()))
  return config_df
 
- def register_pending(self, configurations: pd.DataFrame, context: Optional[pd.DataFrame] = None) -> None:
+ def register_pending(self, *, configurations: pd.DataFrame, context: Optional[pd.DataFrame] = None) -> None:
  raise NotImplementedError()
 
- def surrogate_predict(self, configurations: pd.DataFrame, context: Optional[pd.DataFrame] = None) -> npt.NDArray:
+ def surrogate_predict(self, *, configurations: pd.DataFrame, context: Optional[pd.DataFrame] = None) -> npt.NDArray:
  from smac.utils.configspace import convert_configurations_to_array # pylint: disable=import-outside-toplevel
 
  if context is not None:
@@ -318,11 +318,11 @@ def surrogate_predict(self, configurations: pd.DataFrame, context: Optional[pd.D
  if self.base_optimizer._config_selector._model is None:
  raise RuntimeError('Surrogate model is not yet trained')
 
- configs: npt.NDArray = convert_configurations_to_array(self._to_configspace_configs(configurations))
+ configs: npt.NDArray = convert_configurations_to_array(self._to_configspace_configs(configurations=configurations))
  mean_predictions, _ = self.base_optimizer._config_selector._model.predict(configs)
  return mean_predictions.reshape(-1,)
 
- def acquisition_function(self, configurations: pd.DataFrame, context: Optional[pd.DataFrame] = None) -> npt.NDArray:
+ def acquisition_function(self, *, configurations: pd.DataFrame, context: Optional[pd.DataFrame] = None) -> npt.NDArray:
  if context is not None:
  warn(f"Not Implemented: Ignoring context {list(context.columns)}", UserWarning)
  if self._space_adapter:
@@ -332,15 +332,15 @@ def acquisition_function(self, configurations: pd.DataFrame, context: Optional[p
  if self.base_optimizer._config_selector._acquisition_function is None:
  raise RuntimeError('Acquisition function is not yet initialized')
 
- configs: list = self._to_configspace_configs(configurations)
+ configs: list = self._to_configspace_configs(configurations=configurations)
  return self.base_optimizer._config_selector._acquisition_function(configs).reshape(-1,)
 
  def cleanup(self) -> None:
  if self._temp_output_directory is not None:
  self._temp_output_directory.cleanup()
  self._temp_output_directory = None
 
- def _to_configspace_configs(self, configurations: pd.DataFrame) -> List[ConfigSpace.Configuration]:
+ def _to_configspace_configs(self, *, configurations: pd.DataFrame) -> List[ConfigSpace.Configuration]:
  """Convert a dataframe of configurations to a list of ConfigSpace configurations.
 
  Parameters

mlos_core/mlos_core/optimizers/flaml_optimizer.py

@@ -85,7 +85,7 @@ def __init__(self, *, # pylint: disable=too-many-arguments
  self.evaluated_samples: Dict[ConfigSpace.Configuration, EvaluatedSample] = {}
  self._suggested_config: Optional[dict]
 
- def _register(self, configurations: pd.DataFrame, scores: pd.DataFrame,
+ def _register(self, *, configurations: pd.DataFrame, scores: pd.DataFrame,
  context: Optional[pd.DataFrame] = None) -> None:
  """Registers the given configurations and scores.
 
@@ -112,7 +112,7 @@ def _register(self, configurations: pd.DataFrame, scores: pd.DataFrame,
  score=float(np.average(score.astype(float), weights=self._objective_weights)),
  )
 
- def _suggest(self, context: Optional[pd.DataFrame] = None) -> pd.DataFrame:
+ def _suggest(self, *, context: Optional[pd.DataFrame] = None) -> pd.DataFrame:
  """Suggests a new configuration.
 
  Sampled at random using ConfigSpace.
@@ -132,7 +132,7 @@ def _suggest(self, context: Optional[pd.DataFrame] = None) -> pd.DataFrame:
  config: dict = self._get_next_config()
  return pd.DataFrame(config, index=[0])
 
- def register_pending(self, configurations: pd.DataFrame,
+ def register_pending(self, *, configurations: pd.DataFrame,
  context: Optional[pd.DataFrame] = None) -> None:
  raise NotImplementedError()
 

mlos_core/mlos_core/optimizers/optimizer.py

@@ -68,7 +68,7 @@ def space_adapter(self) -> Optional[BaseSpaceAdapter]:
  """Get the space adapter instance (if any)."""
  return self._space_adapter
 
- def register(self, configurations: pd.DataFrame, scores: pd.DataFrame,
+ def register(self, *, configurations: pd.DataFrame, scores: pd.DataFrame,
  context: Optional[pd.DataFrame] = None) -> None:
  """Wrapper method, which employs the space adapter (if any), before registering the configurations and scores.
 
@@ -101,10 +101,10 @@ def register(self, configurations: pd.DataFrame, scores: pd.DataFrame,
  configurations = self._space_adapter.inverse_transform(configurations)
  assert configurations.shape[1] == len(self.optimizer_parameter_space.values()), \
  "Mismatched configuration shape after inverse transform."
- return self._register(configurations, scores, context)
+ return self._register(configurations=configurations, scores=scores, context=context)
 
  @abstractmethod
- def _register(self, configurations: pd.DataFrame, scores: pd.DataFrame,
+ def _register(self, *, configurations: pd.DataFrame, scores: pd.DataFrame,
  context: Optional[pd.DataFrame] = None) -> None:
  """Registers the given configurations and scores.
 
@@ -120,7 +120,7 @@ def _register(self, configurations: pd.DataFrame, scores: pd.DataFrame,
  """
  pass # pylint: disable=unnecessary-pass # pragma: no cover
 
- def suggest(self, context: Optional[pd.DataFrame] = None, defaults: bool = False) -> pd.DataFrame:
+ def suggest(self, *, context: Optional[pd.DataFrame] = None, defaults: bool = False) -> pd.DataFrame:
  """
  Wrapper method, which employs the space adapter (if any), after suggesting a new configuration.
 
@@ -142,7 +142,7 @@ def suggest(self, context: Optional[pd.DataFrame] = None, defaults: bool = False
  if self.space_adapter is not None:
  configuration = self.space_adapter.inverse_transform(configuration)
  else:
- configuration = self._suggest(context)
+ configuration = self._suggest(context=context)
  assert len(configuration) == 1, \
  "Suggest must return a single configuration."
  assert set(configuration.columns).issubset(set(self.optimizer_parameter_space)), \
@@ -154,7 +154,7 @@ def suggest(self, context: Optional[pd.DataFrame] = None, defaults: bool = False
  return configuration
 
  @abstractmethod
- def _suggest(self, context: Optional[pd.DataFrame] = None) -> pd.DataFrame:
+ def _suggest(self, *, context: Optional[pd.DataFrame] = None) -> pd.DataFrame:
  """Suggests a new configuration.
 
  Parameters
@@ -170,7 +170,7 @@ def _suggest(self, context: Optional[pd.DataFrame] = None) -> pd.DataFrame:
  pass # pylint: disable=unnecessary-pass # pragma: no cover
 
  @abstractmethod
- def register_pending(self, configurations: pd.DataFrame,
+ def register_pending(self, *, configurations: pd.DataFrame,
  context: Optional[pd.DataFrame] = None) -> None:
  """Registers the given configurations as "pending".
  That is it say, it has been suggested by the optimizer, and an experiment trial has been started.
@@ -202,7 +202,7 @@ def get_observations(self) -> Tuple[pd.DataFrame, pd.DataFrame, Optional[pd.Data
  for _, _, context in self._observations]).reset_index(drop=True)
  return (configs, scores, contexts if len(contexts.columns) > 0 else None)
 
- def get_best_observations(self, n_max: int = 1) -> Tuple[pd.DataFrame, pd.DataFrame, Optional[pd.DataFrame]]:
+ def get_best_observations(self, *, n_max: int = 1) -> Tuple[pd.DataFrame, pd.DataFrame, Optional[pd.DataFrame]]:
  """
  Get the N best observations so far as a triplet of DataFrames (config, score, context).
  Default is N=1. The columns are ordered in ASCENDING order of the optimization targets.
@@ -231,7 +231,7 @@ def cleanup(self) -> None:
  Redefine this method in optimizers that require cleanup.
  """
 
- def _from_1hot(self, config: npt.NDArray) -> pd.DataFrame:
+ def _from_1hot(self, *, config: npt.NDArray) -> pd.DataFrame:
  """
  Convert numpy array from one-hot encoding to a DataFrame
  with categoricals and ints in proper columns.
@@ -254,7 +254,7 @@ def _from_1hot(self, config: npt.NDArray) -> pd.DataFrame:
  j += 1
  return pd.DataFrame(df_dict)
 
- def _to_1hot(self, config: Union[pd.DataFrame, pd.Series]) -> npt.NDArray:
+ def _to_1hot(self, *, config: Union[pd.DataFrame, pd.Series]) -> npt.NDArray:
  """
  Convert pandas DataFrame to one-hot-encoded numpy array.
  """

mlos_core/mlos_core/optimizers/random_optimizer.py

@@ -24,7 +24,7 @@ class RandomOptimizer(BaseOptimizer):
  The parameter space to optimize.
  """
 
- def _register(self, configurations: pd.DataFrame, scores: pd.DataFrame,
+ def _register(self, *, configurations: pd.DataFrame, scores: pd.DataFrame,
  context: Optional[pd.DataFrame] = None) -> None:
  """Registers the given configurations and scores.
 
@@ -45,7 +45,7 @@ def _register(self, configurations: pd.DataFrame, scores: pd.DataFrame,
  warn(f"Not Implemented: Ignoring context {list(context.columns)}", UserWarning)
  # should we pop them from self.pending_observations?
 
- def _suggest(self, context: Optional[pd.DataFrame] = None) -> pd.DataFrame:
+ def _suggest(self, *, context: Optional[pd.DataFrame] = None) -> pd.DataFrame:
  """Suggests a new configuration.
 
  Sampled at random using ConfigSpace.
@@ -65,7 +65,7 @@ def _suggest(self, context: Optional[pd.DataFrame] = None) -> pd.DataFrame:
  warn(f"Not Implemented: Ignoring context {list(context.columns)}", UserWarning)
  return pd.DataFrame(dict(self.optimizer_parameter_space.sample_configuration()), index=[0])
 
- def register_pending(self, configurations: pd.DataFrame,
+ def register_pending(self, *, configurations: pd.DataFrame,
  context: Optional[pd.DataFrame] = None) -> None:
  raise NotImplementedError()
  # self._pending_observations.append((configurations, context))

mlos_core/mlos_core/tests/optimizers/bayesian_optimizers_test.py

@@ -39,11 +39,11 @@ def test_context_not_implemented_warning(configuration_space: CS.ConfigurationSp
  context = pd.DataFrame([["something"]])
 
  with pytest.raises(UserWarning):
- optimizer.register(suggestion, scores, context=context)
+ optimizer.register(configurations=suggestion, scores=scores, context=context)
 
  with pytest.raises(UserWarning):
  optimizer.suggest(context=context)
 
  if isinstance(optimizer, BaseBayesianOptimizer):
  with pytest.raises(UserWarning):
- optimizer.surrogate_predict(suggestion, context=context)
+ optimizer.surrogate_predict(configurations=suggestion, context=context)

mlos_core/mlos_core/tests/optimizers/one_hot_test.py

@@ -85,15 +85,15 @@ def test_to_1hot_data_frame(optimizer: BaseOptimizer,
  """
  Toy problem to test one-hot encoding of dataframe.
  """
- assert optimizer._to_1hot(data_frame) == pytest.approx(one_hot_data_frame)
+ assert optimizer._to_1hot(config=data_frame) == pytest.approx(one_hot_data_frame)
 
 
 def test_to_1hot_series(optimizer: BaseOptimizer,
  series: pd.Series, one_hot_series: npt.NDArray) -> None:
  """
  Toy problem to test one-hot encoding of series.
  """
- assert optimizer._to_1hot(series) == pytest.approx(one_hot_series)
+ assert optimizer._to_1hot(config=series) == pytest.approx(one_hot_series)
 
 
 def test_from_1hot_data_frame(optimizer: BaseOptimizer,
@@ -102,7 +102,7 @@ def test_from_1hot_data_frame(optimizer: BaseOptimizer,
  """
  Toy problem to test one-hot decoding of dataframe.
  """
- assert optimizer._from_1hot(one_hot_data_frame).to_dict() == data_frame.to_dict()
+ assert optimizer._from_1hot(config=one_hot_data_frame).to_dict() == data_frame.to_dict()
 
 
 def test_from_1hot_series(optimizer: BaseOptimizer,
@@ -111,7 +111,7 @@ def test_from_1hot_series(optimizer: BaseOptimizer,
  """
  Toy problem to test one-hot decoding of series.
  """
- one_hot_df = optimizer._from_1hot(one_hot_series)
+ one_hot_df = optimizer._from_1hot(config=one_hot_series)
  assert one_hot_df.shape[0] == 1, f"Unexpected number of rows ({one_hot_df.shape[0]} != 1)"
  assert one_hot_df.iloc[0].to_dict() == series.to_dict()
 
@@ -120,7 +120,7 @@ def test_round_trip_data_frame(optimizer: BaseOptimizer, data_frame: pd.DataFram
  """
  Round-trip test for one-hot-encoding and then decoding a data frame.
  """
- df_round_trip = optimizer._from_1hot(optimizer._to_1hot(data_frame))
+ df_round_trip = optimizer._from_1hot(config=optimizer._to_1hot(config=data_frame))
  assert df_round_trip.x.to_numpy() == pytest.approx(data_frame.x)
  assert (df_round_trip.y == data_frame.y).all()
  assert (df_round_trip.z == data_frame.z).all()
@@ -130,7 +130,7 @@ def test_round_trip_series(optimizer: BaseOptimizer, series: pd.DataFrame) -> No
  """
  Round-trip test for one-hot-encoding and then decoding a series.
  """
- series_round_trip = optimizer._from_1hot(optimizer._to_1hot(series))
+ series_round_trip = optimizer._from_1hot(config=optimizer._to_1hot(config=series))
  assert series_round_trip.x.to_numpy() == pytest.approx(series.x)
  assert (series_round_trip.y == series.y).all()
  assert (series_round_trip.z == series.z).all()
@@ -141,7 +141,7 @@ def test_round_trip_reverse_data_frame(optimizer: BaseOptimizer,
  """
  Round-trip test for one-hot-decoding and then encoding of a numpy array.
  """
- round_trip = optimizer._to_1hot(optimizer._from_1hot(one_hot_data_frame))
+ round_trip = optimizer._to_1hot(config=optimizer._from_1hot(config=one_hot_data_frame))
  assert round_trip == pytest.approx(one_hot_data_frame)
 
 
@@ -150,5 +150,5 @@ def test_round_trip_reverse_series(optimizer: BaseOptimizer,
  """
  Round-trip test for one-hot-decoding and then encoding of a numpy array.
  """
- round_trip = optimizer._to_1hot(optimizer._from_1hot(one_hot_series))
+ round_trip = optimizer._to_1hot(config=optimizer._from_1hot(config=one_hot_series))
  assert round_trip == pytest.approx(one_hot_series)

mlos_core/mlos_core/tests/optimizers/optimizer_multiobj_test.py

@@ -99,7 +99,7 @@ def objective(point: pd.DataFrame) -> pd.DataFrame:
  observation = objective(suggestion)
  assert isinstance(observation, pd.DataFrame)
  assert set(observation.columns) == {'main_score', 'other_score'}
- optimizer.register(suggestion, observation)
+ optimizer.register(configurations=suggestion, scores=observation)
 
  (best_config, best_score, best_context) = optimizer.get_best_observations()
  assert isinstance(best_config, pd.DataFrame)