Add multiseries support to graph_prediction_vs_actual_over_time

docs/source/release_notes.rst

@@ -7,6 +7,7 @@ Release Notes
  * Fixes
  * Changes
  * Updated ``ARIMARegressor`` to be compatible with sktime v0.22.0 and beyond :pr:`4283`
+ * Updated ``graph_prediction_vs_actual_over_time()`` to be compatible with multiseries time series :pr:`4284`
  * Documentation Changes
  * Removed erroneous warnings from Data Checks User Guide page and removed ``tqdm`` warning in all notebooks :pr:`4274`
  * Testing Changes

evalml/model_understanding/visualizations.py

@@ -12,6 +12,7 @@
 from evalml.model_family import ModelFamily
 from evalml.objectives.utils import get_objective
 from evalml.problem_types import ProblemTypes
+from evalml.problem_types.utils import is_multiseries
 from evalml.utils import import_or_raise, infer_feature_types, jupyter_check
 
 
@@ -373,25 +374,44 @@
  dates = infer_feature_types(dates)
  prediction = pipeline.predict_in_sample(X, y, X_train=X_train, y_train=y_train)
 
- return pd.DataFrame(
- {
- "dates": dates.reset_index(drop=True),
- "target": y.reset_index(drop=True),
- "prediction": prediction.reset_index(drop=True),
- },
- )
+ if is_multiseries(pipeline.problem_type):
+ return pd.DataFrame(
+ {
+ "dates": dates.reset_index(drop=True),
+ "target": y.reset_index(drop=True),
+ "prediction": prediction.reset_index(drop=True),
+ "series_id": X[pipeline.series_id].reset_index(drop=True),
+ },
+ )
+ else:
+ return pd.DataFrame(
+ {
+ "dates": dates.reset_index(drop=True),
+ "target": y.reset_index(drop=True),
+ "prediction": prediction.reset_index(drop=True),
+ },
+ )
 
 
-def graph_prediction_vs_actual_over_time(pipeline, X, y, X_train, y_train, dates):
+def graph_prediction_vs_actual_over_time(
+ pipeline,
+ X,
+ y,
+ X_train,
+ y_train,
+ dates,
+ single_series=None,
+):
  """Plot the target values and predictions against time on the x-axis.
 
  Args:
  pipeline (TimeSeriesRegressionPipeline): Fitted time series regression pipeline.
- X (pd.DataFrame): Features used to generate new predictions.
- y (pd.Series): Target values to compare predictions against.
+ X (pd.DataFrame): Features used to generate new predictions. If problem is multiseries, X should be stacked.
+ y (pd.Series): Target values to compare predictions against. If problem is multiseries, y should be stacked.
  X_train (pd.DataFrame): Data the pipeline was trained on.
  y_train (pd.Series): Target values for training data.
  dates (pd.Series): Dates corresponding to target values and predictions.
+ single_series (str): The single series that will be plotted from multiseries. Defaults to None
 
  Returns:
  plotly.Figure: Showing the prediction vs actual over time.
@@ -403,8 +423,15 @@
  "plotly.graph_objects",
  error_msg="Cannot find dependency plotly.graph_objects",
  )
+ subplots = import_or_raise(
+ "plotly.subplots",
+ error_msg="Cannot find dependency plotly.subplots",
+ )
 
- if pipeline.problem_type != ProblemTypes.TIME_SERIES_REGRESSION:
+ if (
+ pipeline.problem_type != ProblemTypes.TIME_SERIES_REGRESSION
+ and pipeline.problem_type != ProblemTypes.MULTISERIES_TIME_SERIES_REGRESSION
+ ):
  raise ValueError(
  "graph_prediction_vs_actual_over_time only supports time series regression pipelines! "
  f"Received {str(pipeline.problem_type)}.",
@@ -419,6 +446,53 @@
  dates,
  )
 
+ if is_multiseries(pipeline.problem_type):
+ id_list = (
+ [single_series] if single_series is not None else data["series_id"].unique()
+ )
+ fig = subplots.make_subplots(
+ rows=len(id_list),
+ cols=1,
+ subplot_titles=[f"Series: {id}" for id in id_list],
+ )
+ for curr_count, id in enumerate(id_list):
+ curr_df = data[data["series_id"] == id]
+ fig.append_trace(
+ _go.Scatter(
+ x=curr_df["dates"],
+ y=curr_df["target"],
+ mode="lines+markers",
+ name=f"Series {id}: Target",
+ ),
+ row=curr_count + 1,
+ col=1,
+ )
+ fig.append_trace(
+ _go.Scatter(
+ x=curr_df["dates"],
+ y=curr_df["prediction"],
+ mode="lines+markers",
+ name=f"Series {id}: Prediction",
+ ),
+ row=curr_count + 1,
+ col=1,
+ )
+ fig.update_xaxes(title_text="Time")
+ fig.update_yaxes(title_text=y.name)
+ if single_series is not None:
+ fig.update_layout(
+ height=600,
+ width=1000,
+ title_text=f"Graph for Series {single_series}",
+ )
+ else:
+ fig.update_layout(
+ height=600 + (len(id_list)) * 200,
+ width=1500,
+ title_text="Graph for Multiseries",
+ )
+ return fig
+
  data = [
  _go.Scatter(
  x=data["dates"],

evalml/tests/model_understanding_tests/test_visualizations.py

@@ -28,13 +28,49 @@
  ElasticNetRegressor,
  LinearRegressor,
  MulticlassClassificationPipeline,
+ MultiseriesRegressionPipeline,
  RegressionPipeline,
  TimeSeriesRegressionPipeline,
 )
+from evalml.preprocessing import split_multiseries_data
 from evalml.problem_types import ProblemTypes
 from evalml.utils import get_random_state, infer_feature_types
 
 
+@pytest.fixture(scope="module")
+def component_graph_multiseries():
+ return {
+ "Time Series Featurizer": ["Time Series Featurizer", "X", "y"],
+ "Baseline Multiseries": [
+ "Multiseries Time Series Baseline Regressor",
+ "Time Series Featurizer.x",
+ "y",
+ ],
+ }
+
+
+@pytest.fixture(scope="module")
+def pipeline_parameters_multiseries():
+ return {
+ "pipeline": {
+ "time_index": "date",
+ "max_delay": 10,
+ "forecast_horizon": 7,
+ "gap": 0,
+ "series_id": "series_id",
+ },
+ "Time Series Featurizer": {
+ "time_index": "date",
+ "max_delay": 10,
+ "forecast_horizon": 7,
+ "gap": 0,
+ "delay_features": False,
+ "delay_target": True,
+ },
+ "Baseline Multiseries": {"gap": 0, "forecast_horizon": 7},
+ }
+
+
 @pytest.mark.parametrize("data_type", ["np", "pd", "ww"])
 def test_cost_benefit_matrix_vs_threshold(
  data_type,
@@ -346,6 +382,35 @@
  assert list(results.columns) == ["dates", "target", "prediction"]
 
 
+def test_get_prediction_vs_actual_over_time_data_multiseries(
+ multiseries_ts_data_stacked,
+ component_graph_multiseries,
+ pipeline_parameters_multiseries,
+):
+ X, y = multiseries_ts_data_stacked
+ X_train, _, y_train, _ = split_multiseries_data(
+ X,
+ y,
+ "series_id",
+ "date",
+ )
+ pipeline = MultiseriesRegressionPipeline(
+ component_graph_multiseries,
+ pipeline_parameters_multiseries,
+ )
+ pipeline.fit(X_train, y_train)
+ results = get_prediction_vs_actual_over_time_data(
+ pipeline,
+ X,
+ y,
+ X_train,
+ y_train,
+ pd.Series(X["date"]),
+ )
+ assert isinstance(results, pd.DataFrame)
+ assert list(results.columns) == ["dates", "target", "prediction", "series_id"]
+
+
 def test_graph_prediction_vs_actual_over_time(ts_data, go):
  X, _, y = ts_data()
  X_train, y_train = X.iloc[:30], y.iloc[:30]
@@ -407,6 +472,101 @@
  )
 
 
+def test_graph_prediction_vs_actual_over_time_multiseries_single(
+ multiseries_ts_data_stacked,
+ go,
+ component_graph_multiseries,
+ pipeline_parameters_multiseries,
+):
+ X, y = multiseries_ts_data_stacked
+ X_train, _, y_train, _ = split_multiseries_data(
+ X,
+ y,
+ "series_id",
+ "date",
+ )
+ pipeline = MultiseriesRegressionPipeline(
+ component_graph_multiseries,
+ pipeline_parameters_multiseries,
+ )
+ pipeline.fit(X_train, y_train)
+ fig = graph_prediction_vs_actual_over_time(
+ pipeline,
+ X,
+ y,
+ X_train,
+ y_train,
+ X["date"],
+ "1",
+ )
+ assert isinstance(fig, go.Figure)
+ fig_dict = fig.to_dict()
+
+ assert fig_dict["layout"]["title"]["text"] == "Graph for Series 1"
+ assert fig_dict["layout"]["xaxis"]["title"]["text"] == "Time"
+ assert fig_dict["layout"]["yaxis"]["title"]["text"] == "target"
+ assert len(fig_dict["data"]) == 2
+
+ assert len(fig_dict["data"][0]["x"]) == len(X["date"].unique())
+ assert len(fig_dict["data"][0]["y"]) == len(X["date"].unique())
+ assert not np.isnan(fig_dict["data"][0]["y"]).all()
+ assert fig_dict["data"][0]["name"] == "Series 1: Target"
+
+ assert len(fig_dict["data"][1]["x"]) == len(X["date"].unique())
+ assert len(fig_dict["data"][1]["y"]) == len(X["date"].unique())
+ assert not np.isnan(fig_dict["data"][1]["y"]).all()
+ assert fig_dict["data"][1]["name"] == "Series 1: Prediction"
+
+
+def test_graph_prediction_vs_actual_over_time_multiseries(
+ multiseries_ts_data_stacked,
+ go,
+ component_graph_multiseries,
+ pipeline_parameters_multiseries,
+):
+ X, y = multiseries_ts_data_stacked
+ X_train, _, y_train, _ = split_multiseries_data(
+ X,
+ y,
+ "series_id",
+ "date",
+ )
+ pipeline = MultiseriesRegressionPipeline(
+ component_graph_multiseries,
+ pipeline_parameters_multiseries,
+ )
+ pipeline.fit(X_train, y_train)
+ fig = graph_prediction_vs_actual_over_time(
+ pipeline,
+ X,
+ y,
+ X_train,
+ y_train,
+ X["date"],
+ )
+ assert isinstance(fig, go.Figure)
+
+ fig_dict = fig.to_dict()
+ assert fig_dict["layout"]["title"]["text"] == "Graph for Multiseries"
+ assert fig_dict["layout"]["xaxis"]["title"]["text"] == "Time"
+ assert fig_dict["layout"]["yaxis"]["title"]["text"] == "target"
+
+ # there's 5 series, and each series has two lines (one each for target/prediction)
+ assert len(fig_dict["data"]) == 10
+
+ curr_series = 0
+ for i in range(len(fig_dict["data"])):
+ assert len(fig_dict["data"][i]["x"]) == len(X["date"].unique())
+ assert len(fig_dict["data"][i]["y"]) == len(X["date"].unique())
+ assert not np.isnan(fig_dict["data"][i]["y"]).all()
+
+ if i % 2 == 0:
+ assert fig_dict["data"][i]["name"] == f"Series {curr_series}: Target"
+ else:
+ assert fig_dict["data"][i]["name"] == f"Series {curr_series}: Prediction"
+ curr_series += 1
+
+
 def test_decision_tree_data_from_estimator_not_fitted(tree_estimators):
  est_class, _ = tree_estimators
  with pytest.raises(