Skip to content

Commit

Permalink
add test watch mode (#1728)
Browse files Browse the repository at this point in the history
  • Loading branch information
xuke444 authored Sep 20, 2022
1 parent c496d8a commit c59cc22
Showing 23 changed files with 324 additions and 219 deletions.
2 changes: 1 addition & 1 deletion CONTRIBUTING.md
Original file line number Diff line number Diff line change
@@ -126,7 +126,7 @@ cypress window will open locally - select test file to run the tests
6. `pip install -e raiwidgets` to install raiwidgets locally.
7. `pip install jupyter`
8. `cd notebooks\responsibleaidashboard`
9. To execute tests run `yarn e2e-widget`. Sometimes it is preferable to watch the execution and select only individual test cases. This is possible using `yarn e2e-widget --watch`.
9. To execute tests run `yarn e2e-widget`. Sometimes it is preferable to watch the execution and select only individual test cases. This is possible by running the notebook manually and using `yarn e2e-widget -w --host {host} -n {notebook}` where host is where RAI widget runs on (printed in notebook output) and notebook is the name of the notebook you are running. Eg: `yarn e2e-widget -w --host 5000 -n responsibleaidashboard-census-classification-model-debugging`

Cypress window will open locally - select test file to run the tests.

14 changes: 0 additions & 14 deletions apps/widget-e2e/src/integration/App.spec.ts

This file was deleted.

63 changes: 55 additions & 8 deletions apps/widget/src/app/App.tsx
Original file line number Diff line number Diff line change
@@ -1,29 +1,76 @@
// Copyright (c) Microsoft Corporation.
// Licensed under the MIT License.

import { Spinner } from "@fluentui/react";
import React from "react";

import { config } from "./config";
import { getConfig, IAppConfig } from "./config";
import { ErrorAnalysis } from "./ErrorAnalysis";
import { Fairness } from "./Fairness";
import { Interpret } from "./Interpret";
import { ModelAssessment } from "./ModelAssessment";
import { getModelData } from "./modelData";

export class App extends React.Component {
interface IAppState {
config?: IAppConfig;
modelData?: any;
}
export class App extends React.Component<unknown, IAppState> {
public constructor(props: unknown) {
super(props);
this.state = {};
}
public componentDidMount(): void {
this.loadData();
}
public render(): React.ReactNode {
switch (config.dashboardType) {
if (!this.state.config || !this.state.modelData) {
return <Spinner />;
}
switch (this.state.config.dashboardType) {
case "Fairness":
return <Fairness />;
return (
<Fairness
config={this.state.config}
modelData={this.state.modelData}
/>
);
case "Interpret":
return <Interpret />;
return (
<Interpret
config={this.state.config}
modelData={this.state.modelData}
/>
);
case "ModelPerformance":
return <Interpret dashboardType={config.dashboardType} />;
return (
<Interpret
dashboardType={this.state.config.dashboardType}
config={this.state.config}
modelData={this.state.modelData}
/>
);
case "ErrorAnalysis":
return <ErrorAnalysis />;
return (
<ErrorAnalysis
config={this.state.config}
modelData={this.state.modelData}
/>
);
case "ResponsibleAI":
return <ModelAssessment />;
return (
<ModelAssessment
config={this.state.config}
modelData={this.state.modelData}
/>
);
default:
return "Not Found";
}
}
private loadData = async (): Promise<void> => {
const config = await getConfig();
const modelData = await getModelData();
this.setState({ config, modelData });
};
}
54 changes: 30 additions & 24 deletions apps/widget/src/app/ErrorAnalysis.tsx
Original file line number Diff line number Diff line change
@@ -6,59 +6,65 @@ import { ErrorAnalysisDashboard } from "@responsible-ai/error-analysis";
import React from "react";

import { callFlaskService } from "./callFlaskService";
import { config } from "./config";
import { modelData } from "./modelData";
import { IAppConfig } from "./config";

export class ErrorAnalysis extends React.Component {
interface IErrorAnalysisProps {
config: IAppConfig;
modelData: any;
}
export class ErrorAnalysis extends React.Component<IErrorAnalysisProps> {
public render(): React.ReactNode {
let requestPredictionsMethod = undefined;
let requestMatrixMethod = undefined;
let requestDebugMLMethod = undefined;
let requestImportancesMethod = undefined;
if (config.baseUrl) {
if (modelData.enablePredict) {
if (this.props.config.baseUrl) {
if (this.props.modelData.enablePredict) {
requestPredictionsMethod = async (data: any[]): Promise<any[]> => {
return callFlaskService(data, "/predict");
return callFlaskService(this.props.config, data, "/predict");
};
}
requestMatrixMethod = async (
data: any[]
): Promise<IErrorAnalysisMatrix> => {
return callFlaskService(data, "/matrix");
return callFlaskService(this.props.config, data, "/matrix");
};
requestDebugMLMethod = async (data: any[]): Promise<any[]> => {
return callFlaskService(data, "/tree");
return callFlaskService(this.props.config, data, "/tree");
};
requestImportancesMethod = async (data: any[]): Promise<any[]> => {
return callFlaskService(data, "/importances");
return callFlaskService(this.props.config, data, "/importances");
};
}

return (
<ErrorAnalysisDashboard
modelInformation={{ method: modelData.method, modelClass: "blackbox" }}
modelInformation={{
method: this.props.modelData.method,
modelClass: "blackbox"
}}
dataSummary={{
categoricalMap: modelData.categoricalMap,
classNames: modelData.classNames,
featureNames: modelData.featureNames
categoricalMap: this.props.modelData.categoricalMap,
classNames: this.props.modelData.classNames,
featureNames: this.props.modelData.featureNames
}}
testData={modelData.trainingData}
predictedY={modelData.predictedY}
probabilityY={modelData.probabilityY}
trueY={modelData.trueY}
testData={this.props.modelData.trainingData}
predictedY={this.props.modelData.predictedY}
probabilityY={this.props.modelData.probabilityY}
trueY={this.props.modelData.trueY}
precomputedExplanations={{
ebmGlobalExplanation: modelData.ebmData,
globalFeatureImportance: modelData.globalExplanation,
localFeatureImportance: modelData.localExplanations
ebmGlobalExplanation: this.props.modelData.ebmData,
globalFeatureImportance: this.props.modelData.globalExplanation,
localFeatureImportance: this.props.modelData.localExplanations
}}
requestPredictions={requestPredictionsMethod}
requestDebugML={requestDebugMLMethod}
requestMatrix={requestMatrixMethod}
requestImportances={requestImportancesMethod}
localUrl={config.baseUrl}
locale={config.locale}
features={modelData.featureNames}
errorAnalysisData={modelData.errorAnalysisData}
localUrl={this.props.config.baseUrl}
locale={this.props.config.locale}
features={this.props.modelData.featureNames}
errorAnalysisData={this.props.modelData.errorAnalysisData}
/>
);
}
51 changes: 31 additions & 20 deletions apps/widget/src/app/Fairness.tsx
Original file line number Diff line number Diff line change
@@ -6,41 +6,52 @@ import { FairnessWizard } from "@responsible-ai/fairness";
import React from "react";

import { callFlaskService } from "./callFlaskService";
import { config } from "./config";
import { modelData } from "./modelData";
import { IAppConfig } from "./config";

export class Fairness extends React.Component {
interface IFairnessProps {
config: IAppConfig;
modelData: any;
}
export class Fairness extends React.Component<IFairnessProps> {
public render(): React.ReactNode {
let requestMethod = undefined;
if (config.baseUrl) {
if (this.props.config.baseUrl) {
requestMethod = async (
data: IMetricRequest
): Promise<IMetricResponse> => {
return callFlaskService(data, "/metrics") as Promise<IMetricResponse>;
return callFlaskService(
this.props.config,
data,
"/metrics"
) as Promise<IMetricResponse>;
};
}

return (
<FairnessWizard
dataSummary={{
classNames: modelData.classes,
featureNames: modelData.features
classNames: this.props.modelData.classes,
featureNames: this.props.modelData.features
}}
errorBarsEnabled={modelData.errorBarsEnabled}
testData={modelData.dataset}
predictedY={modelData.predicted_ys}
trueY={modelData.true_y}
modelNames={modelData.model_names}
precomputedMetrics={modelData.precomputedMetrics}
precomputedFeatureBins={modelData.precomputedFeatureBins}
customMetrics={modelData.customMetrics}
predictionType={modelData.predictionType}
errorBarsEnabled={this.props.modelData.errorBarsEnabled}
testData={this.props.modelData.dataset}
predictedY={this.props.modelData.predicted_ys}
trueY={this.props.modelData.true_y}
modelNames={this.props.modelData.model_names}
precomputedMetrics={this.props.modelData.precomputedMetrics}
precomputedFeatureBins={this.props.modelData.precomputedFeatureBins}
customMetrics={this.props.modelData.customMetrics}
predictionType={this.props.modelData.predictionType}
supportedBinaryClassificationPerformanceKeys={
modelData.classification_methods
this.props.modelData.classification_methods
}
supportedRegressionPerformanceKeys={
this.props.modelData.regression_methods
}
supportedProbabilityPerformanceKeys={
this.props.modelData.probability_methods
}
supportedRegressionPerformanceKeys={modelData.regression_methods}
supportedProbabilityPerformanceKeys={modelData.probability_methods}
locale={config.locale}
locale={this.props.config.locale}
requestMetrics={requestMethod}
/>
);
31 changes: 16 additions & 15 deletions apps/widget/src/app/Interpret.tsx
Original file line number Diff line number Diff line change
@@ -5,17 +5,18 @@ import { NewExplanationDashboard } from "@responsible-ai/interpret";
import React from "react";

import { callFlaskService } from "./callFlaskService";
import { config } from "./config";
import { modelData } from "./modelData";
import { IAppConfig } from "./config";
interface IInterpretProps {
dashboardType?: "ModelPerformance";
config: IAppConfig;
modelData: any;
}
export class Interpret extends React.Component<IInterpretProps> {
public render(): React.ReactNode {
let requestMethod = undefined;
if (config.baseUrl) {
if (this.props.config.baseUrl) {
requestMethod = (request: any[]): Promise<any[]> => {
return callFlaskService(request, "/predict");
return callFlaskService(this.props.config, request, "/predict");
};
}

@@ -24,21 +25,21 @@ export class Interpret extends React.Component<IInterpretProps> {
dashboardType={this.props.dashboardType}
modelInformation={{ modelClass: "blackbox" }}
dataSummary={{
classNames: modelData.classNames,
featureNames: modelData.featureNames
classNames: this.props.modelData.classNames,
featureNames: this.props.modelData.featureNames
}}
testData={modelData.trainingData}
predictedY={modelData.predictedY}
probabilityY={modelData.probabilityY}
trueY={modelData.trueY}
testData={this.props.modelData.trainingData}
predictedY={this.props.modelData.predictedY}
probabilityY={this.props.modelData.probabilityY}
trueY={this.props.modelData.trueY}
precomputedExplanations={{
ebmGlobalExplanation: modelData.ebmData,
globalFeatureImportance: modelData.globalExplanation,
localFeatureImportance: modelData.localExplanations
ebmGlobalExplanation: this.props.modelData.ebmData,
globalFeatureImportance: this.props.modelData.globalExplanation,
localFeatureImportance: this.props.modelData.localExplanations
}}
requestPredictions={requestMethod}
locale={config.locale}
explanationMethod={modelData.explanation_method}
locale={this.props.config.locale}
explanationMethod={this.props.modelData.explanation_method}
/>
);
}
Loading

0 comments on commit c59cc22

Please sign in to comment.