Merge branch 'main' into sergiym/opt/configspace-1.0

mlos_bench/mlos_bench/optimizers/mlos_core_optimizer.py

@@ -116,20 +116,19 @@ def bulk_register(
  pd.DataFrame([{} if score is None else score for score in scores])
  )
 
- opt_targets = list(self._opt_targets)
  if status is not None:
  # Select only the completed trials, set scores for failed trials to +inf.
  df_status = pd.Series(status)
  # TODO: Be more flexible with values used for failed trials (not just +inf).
  # Issue: https://github.com/microsoft/MLOS/issues/523
- df_scores.loc[df_status != Status.SUCCEEDED, opt_targets] = float("inf")
+ df_scores[df_status != Status.SUCCEEDED] = float("inf")
  df_status_completed = df_status.apply(Status.is_completed)
  df_configs = df_configs[df_status_completed]
  df_scores = df_scores[df_status_completed]
 
  # TODO: Specify (in the config) which metrics to pass to the optimizer.
  # Issue: https://github.com/microsoft/MLOS/issues/745
- self._opt.register(configs=df_configs, scores=df_scores[opt_targets].astype(float))
+ self._opt.register(configs=df_configs, scores=df_scores)
 
  if _LOG.isEnabledFor(logging.DEBUG):
  (score, _) = self.get_best_observation()
@@ -138,10 +137,19 @@ def bulk_register(
  return True
 
  def _adjust_signs_df(self, df_scores: pd.DataFrame) -> pd.DataFrame:
- """In-place adjust the signs of the scores for MINIMIZATION problem."""
- for opt_target, opt_dir in self._opt_targets.items():
- df_scores[opt_target] *= opt_dir
- return df_scores
+ """Coerce optimization target scores to floats and adjust the signs for
+ MINIMIZATION problem.
+ """
+ df_targets = df_scores[list(self._opt_targets)]
+ try:
+ return df_targets.astype(float) * self._opt_targets.values()
+ except ValueError as ex:
+ _LOG.error(
+ "Some score values cannot be converted to float - check the data ::\n%s",
+ df_targets,
+ exc_info=True,
+ )
+ raise ValueError("Some score values cannot be converted to float") from ex
 
  def _to_df(self, configs: Sequence[Dict[str, TunableValue]]) -> pd.DataFrame:
  """

mlos_bench/mlos_bench/services/remote/azure/azure_auth.py

@@ -66,9 +66,9 @@ def __init__(
  self._access_token = "RENEW *NOW*"
  self._token_expiration_ts = datetime.now(UTC) # Typically, some future timestamp.
 
- # Login as ourselves
- self._cred: Union[azure_id.AzureCliCredential, azure_id.CertificateCredential]
- self._cred = azure_id.AzureCliCredential()
+ # Login as the first identity available, usually ourselves or a managed identity
+ self._cred: Union[azure_id.DefaultAzureCredential, azure_id.CertificateCredential]
+ self._cred = azure_id.DefaultAzureCredential()
 
  # Verify info required for SP auth early
  if "spClientId" in self.config:

mlos_bench/mlos_bench/tests/optimizers/mlos_core_opt_df_test.py

@@ -23,6 +23,10 @@ def mlos_core_optimizer(tunable_groups: TunableGroups) -> MlosCoreOptimizer:
  "optimizer_type": "FLAML",
  "max_suggestions": 10,
  "seed": SEED,
+ "optimization_targets": {
+ "latency": "min",
+ "throughput": "max",
+ },
  }
  return MlosCoreOptimizer(tunable_groups, test_opt_config)
 
@@ -74,3 +78,85 @@ def test_df(mlos_core_optimizer: MlosCoreOptimizer, mock_configs: List[dict]) ->
  "vmSize": "Standard_B2s",
  },
  ]
+
+
+def test_df_str(mlos_core_optimizer: MlosCoreOptimizer, mock_configs: List[dict]) -> None:
+ """Test `MlosCoreOptimizer._to_df()` type coercion on tunables with string
+ values.
+ """
+ df_config_orig = mlos_core_optimizer._to_df(mock_configs)
+ df_config_str = mlos_core_optimizer._to_df(
+ [{key: str(val) for (key, val) in config.items()} for config in mock_configs]
+ )
+ assert df_config_orig.equals(df_config_str)
+
+
+def test_adjust_signs_df(mlos_core_optimizer: MlosCoreOptimizer) -> None:
+ """Test `MlosCoreOptimizer._adjust_signs_df()` on different types of inputs."""
+ df_scores_input = pandas.DataFrame(
+ {
+ "latency": [88.88, 66.66, 99.99, None],
+ "throughput": [111, 222, 333, None],
+ }
+ )
+
+ df_scores_output = pandas.DataFrame(
+ {
+ "latency": [88.88, 66.66, 99.99, float("NaN")],
+ "throughput": [-111, -222, -333, float("NaN")],
+ }
+ )
+
+ # Make sure we adjust the signs for minimization.
+ df_scores = mlos_core_optimizer._adjust_signs_df(df_scores_input)
+ assert df_scores.equals(df_scores_output)
+
+ # Check that the same operation works for string inputs.
+ df_scores = mlos_core_optimizer._adjust_signs_df(df_scores_input.astype(str))
+ assert df_scores.equals(df_scores_output)
+
+
+def test_adjust_signs_df_nan(mlos_core_optimizer: MlosCoreOptimizer) -> None:
+ """Test `MlosCoreOptimizer._adjust_signs_df()` handling None, NaN, and Inf
+ values.
+ """
+ df_scores = mlos_core_optimizer._adjust_signs_df(
+ pandas.DataFrame(
+ {
+ "latency": ["88.88", "NaN", "Inf", "-Inf", None],
+ "throughput": ["111", "NaN", "Inf", "-Inf", None],
+ }
+ )
+ )
+
+ assert df_scores.equals(
+ pandas.DataFrame(
+ {
+ "latency": [88.88, float("NaN"), float("Inf"), float("-Inf"), float("NaN")],
+ "throughput": [-111, float("NaN"), float("-Inf"), float("Inf"), float("NaN")],
+ }
+ )
+ )
+
+
+def test_adjust_signs_df_invalid(mlos_core_optimizer: MlosCoreOptimizer) -> None:
+ """Test `MlosCoreOptimizer._adjust_signs_df()` on invalid inputs."""
+ with pytest.raises(ValueError):
+ mlos_core_optimizer._adjust_signs_df(
+ pandas.DataFrame(
+ {
+ "latency": ["INVALID"],
+ "throughput": ["no input"],
+ }
+ )
+ )
+
+ with pytest.raises(ValueError):
+ mlos_core_optimizer._adjust_signs_df(
+ pandas.DataFrame(
+ {
+ "latency": ["88.88", ""],
+ "throughput": ["111", ""],
+ }
+ )
+ )

mlos_core/mlos_core/tests/optimizers/optimizer_test.py

@@ -157,7 +157,7 @@ def objective(x: pd.Series) -> pd.DataFrame:
 def test_concrete_optimizer_type(optimizer_type: OptimizerType) -> None:
  """Test that all optimizer types are listed in the ConcreteOptimizer constraints."""
  # pylint: disable=no-member
- assert optimizer_type.value in ConcreteOptimizer.__constraints__ # type: ignore[attr-defined]
+ assert optimizer_type.value in ConcreteOptimizer.__constraints__
 
 
 @pytest.mark.parametrize(

mlos_core/mlos_core/tests/spaces/adapters/space_adapter_factory_test.py

@@ -32,10 +32,7 @@
 def test_concrete_optimizer_type(space_adapter_type: SpaceAdapterType) -> None:
  """Test that all optimizer types are listed in the ConcreteOptimizer constraints."""
  # pylint: disable=no-member
- assert (
- space_adapter_type.value
- in ConcreteSpaceAdapter.__constraints__ # type: ignore[attr-defined]
- )
+ assert space_adapter_type.value in ConcreteSpaceAdapter.__constraints__
 
 
 @pytest.mark.parametrize(