Remove functionality for BenchmarkRunner without ground truth (face…

…book#2674)

Summary:
Pull Request resolved: facebook#2674

Context:

This is an alternative to D61431979.

Note: There are benchmarks that do not use `BenchmarkRunner`, but I plan to have them all use `BenchmarkRunner` in the future.

`BenchmarkRunner` technically supports benchmarks without a ground truth, but that functionality is never used, and there aren't any Ax benchmarks that are noisy *and* don't have a ground truth. It is not conceptually clear how such a case should be benchmarked, so it is better to not over-engineer for that need, which may never arise. Instead, benchmarks that lack a ground truth but are deterministic can be treated as noiseless problems with a ground truth, and we can reap support for problems without a ground truth.

Also, `BenchmarkRunner` has some methods that must either be defined or not defined depending on whether there is a ground truth. They can't be abstract because they will not always be defined. With this change, we can make the ground-truth methods abstract and get rid of the rest.

This PR:
- Rewrites docstrings
- Removes method `get_Y_Ystd`
- Makes `get_Y_true` and other methods abstract
- Removes functionality for the case where `get_Y_true` raises a `NotImplementedError`

Reviewed By: ItsMrLin

Differential Revision: D61483962

ax/benchmark/runners/base.py

@@ -5,9 +5,9 @@
 
 # pyre-strict
 
-from abc import ABC, abstractmethod
+from abc import ABC, abstractmethod, abstractproperty
 from math import sqrt
-from typing import Any, Optional, Union
+from typing import Any, Union
 
 import torch
 from ax.core.arm import Arm
@@ -21,45 +21,44 @@
 
 
 class BenchmarkRunner(Runner, ABC):
-
- @property
- @abstractmethod
+ """
+ A Runner that produces both observed and ground-truth values.
+
+ Observed values equal ground-truth values plus noise, with the noise added
+ according to the standard deviations returned by `get_noise_stds()`.
+
+ This runner does require that every benchmark has a ground truth, which
+ won't necessarily be true for real-world problems. Such problems fall into
+ two categories:
+ - If they are deterministic, they can be used with this runner by
+ viewing them as noiseless problems where the observed values are the
+ ground truth. The observed values will be used for tracking the
+ progress of optimization.
+ - If they are not deterministc, they are not supported. It is not
+ conceptually clear how to benchmark such problems, so we decided to
+ not over-engineer for that before such a use case arrives.
+ """
+
+ @abstractproperty
  def outcome_names(self) -> list[str]:
  """The names of the outcomes of the problem (in the order of the outcomes)."""
  pass # pragma: no cover
 
  def get_Y_true(self, arm: Arm) -> Tensor:
- """Function returning the ground truth values for a given arm. The
- synthetic noise is added as part of the Runner's `run()` method.
- For problems that do not have a ground truth, the Runner must
- implement the `get_Y_Ystd()` method instead."""
- raise NotImplementedError(
- "Must implement method `get_Y_true()` for Runner "
- f"{self.__class__.__name__} as it does not implement a "
- "`get_Y_Ystd()` method."
- )
+ """
+ Return the ground truth values for a given arm.
+
+ Synthetic noise is added as part of the Runner's `run()` method.
+ """
+ ...
 
+ @abstractmethod
  def get_noise_stds(self) -> Union[None, float, dict[str, float]]:
- """Function returning the standard errors for the synthetic noise
- to be applied to the observed values. For problems that do not have
- a ground truth, the Runner must implement the `get_Y_Ystd()` method
- instead."""
- raise NotImplementedError(
- "Must implement method `get_Y_Ystd()` for Runner "
- f"{self.__class__.__name__} as it does not implement a "
- "`get_noise_stds()` method."
- )
-
- def get_Y_Ystd(self, arm: Arm) -> tuple[Tensor, Optional[Tensor]]:
- """Function returning the observed values and their standard errors
- for a given arm. This function is unused for problems that have a
- ground truth (in this case `get_Y_true()` is used), and is required
- for problems that do not have a ground truth."""
- raise NotImplementedError(
- "Must implement method `get_Y_Ystd()` for Runner "
- f"{self.__class__.__name__} as it does not implement a "
- "`get_Y_true()` method."
- )
+ """
+ Return the standard errors for the synthetic noise to be applied to the
+ observed values.
+ """
+ ...
 
  def run(self, trial: BaseTrial) -> dict[str, Any]:
  """Run the trial by evaluating its parameterization(s).
@@ -110,33 +109,26 @@ def run(self, trial: BaseTrial) -> dict[str, Any]:
  )
 
  for arm in trial.arms:
- try:
- # Case where we do have a ground truth
- Y_true = self.get_Y_true(arm)
- Ys_true[arm.name] = Y_true.tolist()
- if noise_stds is None:
- # No noise, so just return the true outcome.
- Ystds[arm.name] = [0.0] * len(Y_true)
- Ys[arm.name] = Y_true.tolist()
- else:
- # We can scale the noise std by the inverse of the relative sample
- # budget allocation to each arm. This works b/c (i) we assume that
- # observations per unit sample budget are i.i.d. and (ii) the
- # normalized weights sum to one.
- std = noise_stds_tsr.to(Y_true) / sqrt(nlzd_arm_weights[arm])
- Ystds[arm.name] = std.tolist()
- Ys[arm.name] = (Y_true + std * torch.randn_like(Y_true)).tolist()
- except NotImplementedError:
- # Case where we don't have a ground truth.
- Y, Ystd = self.get_Y_Ystd(arm)
- Ys[arm.name] = Y.tolist()
- Ystds[arm.name] = Ystd.tolist() if Ystd is not None else None
+ # Case where we do have a ground truth
+ Y_true = self.get_Y_true(arm)
+ Ys_true[arm.name] = Y_true.tolist()
+ if noise_stds is None:
+ # No noise, so just return the true outcome.
+ Ystds[arm.name] = [0.0] * len(Y_true)
+ Ys[arm.name] = Y_true.tolist()
+ else:
+ # We can scale the noise std by the inverse of the relative sample
+ # budget allocation to each arm. This works b/c (i) we assume that
+ # observations per unit sample budget are i.i.d. and (ii) the
+ # normalized weights sum to one.
+ std = noise_stds_tsr.to(Y_true) / sqrt(nlzd_arm_weights[arm])
+ Ystds[arm.name] = std.tolist()
+ Ys[arm.name] = (Y_true + std * torch.randn_like(Y_true)).tolist()
 
  run_metadata = {
  "Ys": Ys,
  "Ystds": Ystds,
  "outcome_names": self.outcome_names,
+ "Ys_true": Ys_true,
  }
- if Ys_true: # only add key if we actually have a ground truth
- run_metadata["Ys_true"] = Ys_true
  return run_metadata