Add forecasting dashboard - UI - time series selection & forecast comparison

.eslintrc/.eslintrc.custom.eslintrc

@@ -123,6 +123,7 @@
           "target_column",
           "task_type",
           "test_data",
+          "time_series_id_column_names",
           "treatment_feature",
           "treatment_gains",
           "tree_features",

apps/dashboard/src/model-assessment-forecasting/App.tsx

@@ -2,7 +2,6 @@
 // Licensed under the MIT License.
 
 import { ITheme } from "@fluentui/react";
-import { HelpMessageDict } from "@responsible-ai/error-analysis";
 import { Language } from "@responsible-ai/localization";
 import {
   ModelAssessmentDashboard,
@@ -11,6 +10,12 @@ import {
 } from "@responsible-ai/model-assessment";
 import React from "react";
 
+import {
+  bobsSandwichesSandwich,
+  giorgiosPizzeriaBoston,
+  nonnasCannoliBoston
+} from "./__mock_data__/mockForecastingData";
+
 interface IAppProps extends IModelAssessmentData {
   theme: ITheme;
   language: Language;
@@ -20,28 +25,62 @@ interface IAppProps extends IModelAssessmentData {
 }
 
 export class App extends React.Component<IAppProps> {
-  private messages: HelpMessageDict = {
-    LocalExpAndTestReq: [{ displayText: "LocalExpAndTestReq", format: "text" }],
-    LocalOrGlobalAndTestReq: [
-      { displayText: "LocalOrGlobalAndTestReq", format: "text" }
-    ],
-    PredictorReq: [{ displayText: "PredictorReq", format: "text" }],
-    TestReq: [{ displayText: "TestReq", format: "text" }]
-  };
-
   public render(): React.ReactNode {
     this.props.modelExplanationData?.forEach(
       (modelExplanationData) => (modelExplanationData.modelClass = "blackbox")
     );
     const modelAssessmentDashboardProps: IModelAssessmentDashboardProps = {
       ...this.props,
-      cohortData: [],
+      cohortData: [
+        giorgiosPizzeriaBoston,
+        nonnasCannoliBoston,
+        bobsSandwichesSandwich
+      ],
       locale: this.props.language,
       localUrl: "https://www.bing.com/",
-      stringParams: { contextualHelp: this.messages },
-      theme: this.props.theme
+      requestForecast: this.requestForecast
     };
 
     return <ModelAssessmentDashboard {...modelAssessmentDashboardProps} />;
   }
+
+  private requestForecast = (
+    x: any[],
+    abortSignal: AbortSignal
+  ): Promise<any[]> => {
+    return new Promise<number[]>((resolver) => {
+      setTimeout(() => {
+        if (abortSignal.aborted) {
+          return;
+        }
+        let start: number;
+        let end: number;
+        if (x[0][0].arg[0] === 1) {
+          // Giorgio's pizzeria
+          start = 0;
+          end = 10;
+        } else if (x[0][0].arg[0] === 0) {
+          // Bob's sandwiches
+          start = 10;
+          end = 20;
+        } else {
+          // Nonna's cannolis
+          start = 20;
+          end = 30;
+        }
+        const preds = this.props.dataset.predicted_y?.slice(
+          start,
+          end
+        ) as number[];
+        if (x[2].length === 0) {
+          // return original predictions
+          resolver(preds);
+        } else {
+          // return predictions based on modified features
+          // we have to mock this part since we don't have a model available
+          resolver(preds.map((p) => p + 200 * (Math.random() - 0.5)));
+        }
+      }, 300);
+    });
+  };
 }

apps/dashboard/src/model-assessment-forecasting/__mock_data__/mockForecastingData.ts

@@ -1,17 +1,152 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-import { DatasetTaskType, IDataset } from "@responsible-ai/core-ui";
+import {
+  DatasetTaskType,
+  FilterMethods,
+  IDataset,
+  IPreBuiltCohort
+} from "@responsible-ai/core-ui";
+
+export const giorgiosPizzeriaBoston: IPreBuiltCohort = {
+  cohort_filter_list: [
+    {
+      arg: ["Giorgio's pizzeria"],
+      column: "restaurant",
+      method: FilterMethods.Includes
+    },
+    {
+      arg: ["Boston, MA"],
+      column: "city",
+      method: FilterMethods.Includes
+    }
+  ],
+  name: "restaurant = Giorgio's pizzeria, city = Boston, MA"
+};
+
+export const nonnasCannoliBoston: IPreBuiltCohort = {
+  cohort_filter_list: [
+    {
+      arg: ["Nonna's cannoli"],
+      column: "restaurant",
+      method: FilterMethods.Includes
+    },
+    {
+      arg: ["Boston, MA"],
+      column: "city",
+      method: FilterMethods.Includes
+    }
+  ],
+  name: "restaurant = Nonna's cannoli, city = Boston, MA"
+};
+
+export const bobsSandwichesSandwich: IPreBuiltCohort = {
+  cohort_filter_list: [
+    {
+      arg: ["Bob's sandwiches"],
+      column: "restaurant",
+      method: FilterMethods.Includes
+    },
+    {
+      arg: ["Sandwich, MA"],
+      column: "city",
+      method: FilterMethods.Includes
+    }
+  ],
+  name: "restaurant = Bob's sandwiches, city = Sandwich, MA"
+};
 
 // Based on how much money is spent on ads and the daily outside temperature
 // predict the number of people dining at a restaurant on any given day.
 export const mockForecastingData: IDataset = {
-  categorical_features: [],
-  feature_metadata: {},
-  feature_names: [],
+  categorical_features: ["restaurant", "city"],
+  feature_metadata: {
+    categorical_features: ["restaurant", "city"],
+    time_series_id_column_names: ["restaurant", "city"]
+  },
+  feature_names: ["ads", "temperature", "restaurant", "city"],
 
-  features: [],
-  predicted_y: [],
-  task_type: DatasetTaskType.Regression,
-  true_y: []
+  features: [
+    [1, 56, "Giorgio's pizzeria", "Boston, MA"],
+    [2, 65, "Giorgio's pizzeria", "Boston, MA"],
+    [1.3, 43, "Giorgio's pizzeria", "Boston, MA"],
+    [2.1, 55, "Giorgio's pizzeria", "Boston, MA"],
+    [1.6, 70, "Giorgio's pizzeria", "Boston, MA"],
+    [1.9, 67, "Giorgio's pizzeria", "Boston, MA"],
+    [1.3, 84, "Giorgio's pizzeria", "Boston, MA"],
+    [2.4, 76, "Giorgio's pizzeria", "Boston, MA"],
+    [1.9, 73, "Giorgio's pizzeria", "Boston, MA"],
+    [2.9, 61, "Giorgio's pizzeria", "Boston, MA"],
+    [0.2, 56, "Nonna's cannoli", "Boston, MA"],
+    [0.1, 65, "Nonna's cannoli", "Boston, MA"],
+    [0.4, 43, "Nonna's cannoli", "Boston, MA"],
+    [0.3, 55, "Nonna's cannoli", "Boston, MA"],
+    [0.2, 70, "Nonna's cannoli", "Boston, MA"],
+    [0.1, 67, "Nonna's cannoli", "Boston, MA"],
+    [0.3, 84, "Nonna's cannoli", "Boston, MA"],
+    [0.4, 76, "Nonna's cannoli", "Boston, MA"],
+    [0.3, 73, "Nonna's cannoli", "Boston, MA"],
+    [0.5, 61, "Nonna's cannoli", "Boston, MA"],
+    [3, 27, "Bob's sandwiches", "Sandwich, MA"],
+    [2.5, 31, "Bob's sandwiches", "Sandwich, MA"],
+    [2.7, 33, "Bob's sandwiches", "Sandwich, MA"],
+    [3.9, 47, "Bob's sandwiches", "Sandwich, MA"],
+    [3.4, 91, "Bob's sandwiches", "Sandwich, MA"],
+    [3.1, 87, "Bob's sandwiches", "Sandwich, MA"],
+    [1.9, 81, "Bob's sandwiches", "Sandwich, MA"],
+    [1.8, 34, "Bob's sandwiches", "Sandwich, MA"],
+    [3.4, 53, "Bob's sandwiches", "Sandwich, MA"],
+    [3, 62, "Bob's sandwiches", "Sandwich, MA"]
+  ],
+  index: [
+    "10-10-2022",
+    "10-11-2022",
+    "10-12-2022",
+    "10-13-2022",
+    "10-14-2022",
+    "10-15-2022",
+    "10-16-2022",
+    "10-17-2022",
+    "10-18-2022",
+    "10-19-2022",
+    "10-10-2022",
+    "10-11-2022",
+    "10-12-2022",
+    "10-13-2022",
+    "10-14-2022",
+    "10-15-2022",
+    "10-16-2022",
+    "10-17-2022",
+    "10-18-2022",
+    "10-19-2022",
+    "10-10-2022",
+    "10-11-2022",
+    "10-12-2022",
+    "10-13-2022",
+    "10-14-2022",
+    "10-15-2022",
+    "10-16-2022",
+    "10-17-2022",
+    "10-18-2022",
+    "10-19-2022",
+    "10-10-2022",
+    "10-11-2022",
+    "10-12-2022",
+    "10-13-2022",
+    "10-14-2022",
+    "10-15-2022",
+    "10-16-2022",
+    "10-17-2022",
+    "10-18-2022",
+    "10-19-2022"
+  ],
+  predicted_y: [
+    213, 349, 320, 303, 511, 501, 762, 631, 599, 398, 243, 549, 390, 301, 311,
+    701, 722, 681, 299, 498, 763, 149, 120, 103, 111, 101, 162, 131, 299, 198
+  ],
+  task_type: DatasetTaskType.Forecasting,
+  true_y: [
+    240, 310, 342, 392, 514, 501, 795, 621, 600, 422, 222, 500, 345, 678, 343,
+    454, 667, 399, 588, 440, 120, 99, 101, 110, 150, 130, 125, 127, 200, 187
+  ]
 };

apps/widget/src/app/ModelAssessment.tsx

@@ -36,6 +36,7 @@ export class ModelAssessment extends React.Component<IModelAssessmentProps> {
       | "requestBoxPlotDistribution"
       | "requestDatasetAnalysisBarChart"
       | "requestDatasetAnalysisBoxChart"
+      | "requestForecast"
       | "requestGlobalCausalEffects"
       | "requestGlobalCausalPolicy"
       | "requestGlobalExplanations"
@@ -85,6 +86,9 @@ export class ModelAssessment extends React.Component<IModelAssessmentProps> {
           "/model_overview_probability_distribution"
         );
       };
+      callBack.requestForecast = async (data: any[]): Promise<any[]> => {
+        return callFlaskService(this.props.config, data, "/forecast");
+      };
       callBack.requestGlobalCausalEffects = async (
         id: string,
         filter: unknown[],

libs/core-ui/src/index.ts

@@ -2,6 +2,7 @@
 // Licensed under the MIT License.
 
 export * from "./lib/cohortKey";
+export * from "./lib/Cohort/isAllDataCohort";
 export * from "./lib/Cohort/Cohort";
 export * from "./lib/Cohort/CohortList/CohortList";
 export * from "./lib/Cohort/Constants";
@@ -42,6 +43,7 @@ export * from "./lib/util/Never";
 export * from "./lib/util/PartialRequired";
 export * from "./lib/util/nameof";
 export * from "./lib/util/rowErrorSize";
+export * from "./lib/util/TimeUtils";
 export * from "./lib/util/getBoxData";
 export * from "./lib/util/getBasicFilterString";
 export * from "./lib/util/getCommonStyles";

libs/core-ui/src/lib/Cohort/Cohort.ts

@@ -15,7 +15,8 @@ export enum CohortSource {
   None = "None",
   TreeMap = "Tree map",
   HeatMap = "Heat map",
-  ManuallyCreated = "Manually created"
+  ManuallyCreated = "Manually created",
+  Prebuilt = "Prebuilt"
 }
 
 export class Cohort {

libs/core-ui/src/lib/Cohort/CohortInfo/CohortInfo.tsx

@@ -8,6 +8,7 @@ import React from "react";
 import { getCohortFilterCount } from "../../util/getCohortFilterCount";
 import { ErrorCohortStats } from "../CohortStats";
 import { ErrorCohort } from "../ErrorCohort";
+import { isAllDataErrorCohort } from "../isAllDataCohort";
 import { PredictionPath } from "../PredictionPath/PredictionPath";
 
 import { cohortInfoStyles } from "./CohortInfo.styles";
@@ -37,8 +38,7 @@ export class CohortInfo extends React.PureComponent<ICohortInfoProps> {
             <Label>
               {localization.ErrorAnalysis.CohortInfo.basicInformation}
             </Label>
-            {this.props.currentCohort.cohort.name !==
-              localization.ErrorAnalysis.Cohort.defaultLabel && (
+            {!isAllDataErrorCohort(this.props.currentCohort, true) && (
               <Text>{this.props.currentCohort.cohort.name}</Text>
             )}
             <Text>

libs/core-ui/src/lib/Cohort/CohortInfoSection/CohortInfoSection.tsx

@@ -10,6 +10,7 @@ import {
   IModelAssessmentContext,
   ModelAssessmentContext
 } from "../../Context/ModelAssessmentContext";
+import { isAllDataErrorCohort } from "../isAllDataCohort";
 
 export interface ICohortInfoSectionProps {
   toggleShiftCohortVisibility: () => void;
@@ -26,10 +27,7 @@ export class CohortInfoSection extends React.PureComponent<ICohortInfoSectionPro
     // add (default) if it's the default cohort
     let cohortInfoTitle =
       localization.ModelAssessment.CohortInformation.GlobalCohort + cohortName;
-    if (
-      currentCohort.cohort.filters.length === 0 &&
-      currentCohort.cohort.name === localization.Interpret.Cohort.defaultLabel
-    ) {
+    if (isAllDataErrorCohort(currentCohort, true)) {
       cohortInfoTitle +=
         localization.ModelAssessment.CohortInformation.DefaultCohort;
     }

libs/core-ui/src/lib/Cohort/isAllDataCohort.ts

@@ -0,0 +1,25 @@
+// Copyright (c) Microsoft Corporation.
+// Licensed under the MIT License.
+
+import { localization } from "@responsible-ai/localization";
+
+import { Cohort } from "./Cohort";
+import { ErrorCohort } from "./ErrorCohort";
+
+export function isAllDataCohort(cohort: Cohort, checkName?: boolean): boolean {
+  // Comparing localized strings such as the cohort name is bad.
+  // Only comparing the filters is not enough, though, because there
+  // could be multiple cohorts without filters.
+  return (
+    cohort.filters.length === 0 &&
+    cohort.compositeFilters.length === 0 &&
+    (!checkName || cohort.name === localization.Interpret.Cohort.defaultLabel)
+  );
+}
+
+export function isAllDataErrorCohort(
+  errorCohort: ErrorCohort,
+  checkName?: boolean
+): boolean {
+  return isAllDataCohort(errorCohort.cohort, checkName);
+}

libs/core-ui/src/lib/Context/ModelAssessmentContext.tsx

@@ -112,6 +112,10 @@ export interface IModelAssessmentContext {
   requestExp?:
     | ((index: number, abortSignal: AbortSignal) => Promise<any[]>)
     | undefined;
+  requestForecast?: (
+    request: any[],
+    abortSignal: AbortSignal
+  ) => Promise<number[]>;
   shiftErrorCohort(cohort: ErrorCohort): void;
   addCohort(cohort: Cohort, switchNew?: boolean): void;
   editCohort(cohort: Cohort, switchNew?: boolean): void;