Add evaluate_oracle method to BenchmarkRunner (#2705)

Summary:
Pull Request resolved: #2705

Docstring should make this self-explanatory.

In addition to the increased ease of development, enables reaping a lot of LOC in D61415525

Reviewed By: Balandat

Differential Revision: D61431980

ax/benchmark/runners/base.py

@@ -51,6 +51,22 @@ def get_Y_true(self, arm: Arm) -> Tensor:
         """
         ...
 
+    def evaluate_oracle(self, arm: Arm) -> Tensor:
+        """
+        Evaluate oracle metric values at a parameterization. In the base class,
+        oracle=ground truth.
+
+        This method can be customized for more complex setups based on different
+        notions of what the "oracle" value should be. For example, in a
+        multi-task or multi-fidelity problem, it might be appropriate to
+        evaluate at the target task or fidelity. In a simple single-task
+        single-fidelity problem, this could the ground truth if available or the
+        "Y" value if the ground truth is not available. With a
+        preference-learned objective, the values might be true metrics evaluated
+        at the true utility function (which would be unobserved in reality).
+        """
+        return self.get_Y_true(arm=arm)
+
     @abstractmethod
     def get_noise_stds(self) -> Union[None, float, dict[str, float]]:
         """

ax/benchmark/tests/runners/test_botorch_test_problem.py

@@ -150,6 +150,8 @@ def test_synthetic_runner(self) -> None:
                         torch.Size([2]), X.pow(2).sum().item(), dtype=torch.double
                     )
                 self.assertTrue(torch.allclose(Y, expected_Y))
+                oracle = runner.evaluate_oracle(arm=arm)
+                self.assertTrue(torch.equal(Y, oracle))
 
             with self.subTest(f"test `run()`, {test_description}"):
                 trial = Mock(spec=Trial)