Log an exception rather than erroring out when data is found for metr…

…ics that are not attached to the experiment

Summary: D57165887 added an error when modeling layer encountered data for metrics that are not attached to the experiment. Daniel has been seeing these errors in some use cases where it is non-trivial to attach these metrics to the experiment. In order to unblock, we are relaxing the error, and logging an exception instead. This should keep the issue visible while letting the experiment proceed.

Reviewed By: danielcohenlive

Differential Revision: D61544922

ax/core/observation.py

@@ -12,10 +12,10 @@
 import warnings
 from collections.abc import Iterable
 from copy import deepcopy
+from logging import Logger
 from typing import Optional
 
 import ax.core.experiment as experiment
-
 import numpy as np
 import pandas as pd
 from ax.core.arm import Arm
@@ -25,11 +25,12 @@
 from ax.core.map_data import MapData
 from ax.core.map_metric import MapMetric
 from ax.core.types import TCandidateMetadata, TParameterization
-from ax.exceptions.core import UserInputError
 from ax.utils.common.base import Base
 from ax.utils.common.constants import Keys
+from ax.utils.common.logger import get_logger
 from ax.utils.common.typeutils import checked_cast, not_none
 
+logger: Logger = get_logger(__name__)
 
 TIME_COLS = {"start_time", "end_time"}
 
@@ -385,12 +386,14 @@ def _filter_data_on_status(
         metric_name = g
         if metric_name not in experiment.metrics:
             # Observations can only be made for metrics attached to the experiment.
-            raise UserInputError(
+            logger.exception(
                 f"Data contains metric {metric_name} that has not been added to the "
                 "experiment. You can either update the `optimization_config` or attach "
                 "it as a tracking metric using `Experiment.add_tracking_metrics` "
-                "or `AxClient.add_tracking_metrics`."
+                "or `AxClient.add_tracking_metrics`. Ignoring all data for "
+                f"metric {metric_name}."
             )
+            continue
         metric = experiment.metrics[metric_name]
         statuses_to_include_metric = (
             statuses_to_include_map_metric

ax/core/tests/test_observation.py

@@ -7,7 +7,7 @@
 # pyre-strict
 
 import json
-from unittest.mock import Mock, PropertyMock
+from unittest.mock import Mock, patch, PropertyMock
 
 import numpy as np
 import pandas as pd
@@ -30,7 +30,6 @@
 )
 from ax.core.trial import Trial
 from ax.core.types import TParameterization
-from ax.exceptions.core import UserInputError
 from ax.utils.common.testutils import TestCase
 from ax.utils.common.typeutils import not_none
 
@@ -615,8 +614,11 @@ def test_ObservationsFromDataAbandoned(self) -> None:
         data = Data(df=df)
 
         # Data includes metric "c" not attached to the experiment.
-        with self.assertRaisesRegex(UserInputError, "Data contains metric"):
+        with patch("ax.core.observation.logger.exception") as mock_logger:
             observations_from_data(experiment, data)
+        mock_logger.assert_called_once()
+        call_str = mock_logger.call_args.args[0]
+        self.assertIn("Data contains metric c that has not been", call_str)
 
         # Add "c" to the experiment
         type(experiment).metrics = PropertyMock(