Log an exception rather than erroring out when data is found for metrics that are not attached to the experiment

ax/core/observation.py

@@ -12,10 +12,10 @@
 import warnings
 from collections.abc import Iterable
 from copy import deepcopy
+from logging import Logger
 from typing import Optional
 
 import ax.core.experiment as experiment
-
 import numpy as np
 import pandas as pd
 from ax.core.arm import Arm
@@ -25,11 +25,12 @@
 from ax.core.map_data import MapData
 from ax.core.map_metric import MapMetric
 from ax.core.types import TCandidateMetadata, TParameterization
-from ax.exceptions.core import UserInputError
 from ax.utils.common.base import Base
 from ax.utils.common.constants import Keys
+from ax.utils.common.logger import get_logger
 from ax.utils.common.typeutils import checked_cast, not_none
 
+logger: Logger = get_logger(__name__)
 
 TIME_COLS = {"start_time", "end_time"}
 
@@ -385,12 +386,14 @@ def _filter_data_on_status(
         metric_name = g
         if metric_name not in experiment.metrics:
             # Observations can only be made for metrics attached to the experiment.
-            raise UserInputError(
+            logger.exception(
                 f"Data contains metric {metric_name} that has not been added to the "
                 "experiment. You can either update the `optimization_config` or attach "
                 "it as a tracking metric using `Experiment.add_tracking_metrics` "
-                "or `AxClient.add_tracking_metrics`."
+                "or `AxClient.add_tracking_metrics`. Ignoring all data for "
+                f"metric {metric_name}."
             )
+            continue
         metric = experiment.metrics[metric_name]
         statuses_to_include_metric = (
             statuses_to_include_map_metric

ax/core/tests/test_observation.py

@@ -7,7 +7,7 @@
 # pyre-strict
 
 import json
-from unittest.mock import Mock, PropertyMock
+from unittest.mock import Mock, patch, PropertyMock
 
 import numpy as np
 import pandas as pd
@@ -30,7 +30,6 @@
 )
 from ax.core.trial import Trial
 from ax.core.types import TParameterization
-from ax.exceptions.core import UserInputError
 from ax.utils.common.testutils import TestCase
 from ax.utils.common.typeutils import not_none
 
@@ -615,8 +614,11 @@ def test_ObservationsFromDataAbandoned(self) -> None:
         data = Data(df=df)
 
         # Data includes metric "c" not attached to the experiment.
-        with self.assertRaisesRegex(UserInputError, "Data contains metric"):
+        with patch("ax.core.observation.logger.exception") as mock_logger:
             observations_from_data(experiment, data)
+        mock_logger.assert_called_once()
+        call_str = mock_logger.call_args.args[0]
+        self.assertIn("Data contains metric c that has not been", call_str)
 
         # Add "c" to the experiment
         type(experiment).metrics = PropertyMock(

ax/service/tests/scheduler_test_utils.py

@@ -2481,10 +2481,12 @@ def test_generate_candidates_does_not_generate_if_overconstrained(self) -> None:
         self.branin_experiment.optimization_config.outcome_constraints = [
             get_outcome_constraint(
                 metric=get_branin_metric(name="branin_constraint"),
-                bound=20,
+                bound=20.0,
                 relative=True,
             )
         ]
+        if self.ALWAYS_USE_DB:
+            save_experiment(self.branin_experiment, config=self.db_config)
         self.assertTrue(scheduler.experiment.lookup_data().df.empty)
 
         # WHEN generating candidates