From ea3917fab28604041075d97ad2bf2c3e56a08e91 Mon Sep 17 00:00:00 2001 From: Riley Jones Date: Thu, 13 Apr 2023 22:39:07 +0000 Subject: [PATCH] port runs data source to oss --- tensorboard/webapp/runs/data_source/BUILD | 2 + .../runs/data_source/runs_data_source.ts | 169 ++++++++++- .../runs/data_source/runs_data_source_test.ts | 276 +++++++++++++++++- .../webapp/runs/data_source/testing.ts | 199 ++++++++++++- 4 files changed, 635 insertions(+), 11 deletions(-) diff --git a/tensorboard/webapp/runs/data_source/BUILD b/tensorboard/webapp/runs/data_source/BUILD index ccf9384937..b35f4e4705 100644 --- a/tensorboard/webapp/runs/data_source/BUILD +++ b/tensorboard/webapp/runs/data_source/BUILD @@ -48,7 +48,9 @@ tf_ts_library( "runs_data_source_test.ts", ], deps = [ + ":backend_types", ":data_source", + ":testing", "//tensorboard/webapp/angular:expect_angular_core_testing", "//tensorboard/webapp/webapp_data_source:http_client_testing", "@npm//@types/jasmine", diff --git a/tensorboard/webapp/runs/data_source/runs_data_source.ts b/tensorboard/webapp/runs/data_source/runs_data_source.ts index 6c7c309d73..854caabc33 100644 --- a/tensorboard/webapp/runs/data_source/runs_data_source.ts +++ b/tensorboard/webapp/runs/data_source/runs_data_source.ts @@ -13,21 +13,71 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ import {Injectable} from '@angular/core'; -import {Observable, of} from 'rxjs'; -import {map} from 'rxjs/operators'; -import {TBHttpClient} from '../../webapp_data_source/tb_http_client'; +import {Observable, of, throwError} from 'rxjs'; +import {catchError, map, mergeMap} from 'rxjs/operators'; import { + HttpErrorResponse, + TBHttpClient, +} from '../../webapp_data_source/tb_http_client'; +import * as backendTypes from './runs_backend_types'; +import { + Domain, + DomainType, HparamsAndMetadata, + HparamSpec, + HparamValue, + MetricSpec, Run, RunsDataSource, + RunToHparamsAndMetrics, } from './runs_data_source_types'; +const HPARAMS_HTTP_PATH_PREFIX = 'data/plugin/hparams'; + type BackendGetRunsResponse = string[]; function runToRunId(run: string, experimentId: string) { return `${experimentId}/${run}`; } +function transformBackendHparamSpec( + hparamInfo: backendTypes.HparamSpec +): HparamSpec { + let domain: Domain; + if (backendTypes.isDiscreteDomainHparamSpec(hparamInfo)) { + domain = {type: DomainType.DISCRETE, values: hparamInfo.domainDiscrete}; + } else if (backendTypes.isIntervalDomainHparamSpec(hparamInfo)) { + domain = {...hparamInfo.domainInterval, type: DomainType.INTERVAL}; + } else { + domain = { + type: DomainType.INTERVAL, + minValue: -Infinity, + maxValue: Infinity, + }; + } + return { + description: hparamInfo.description, + displayName: hparamInfo.displayName, + name: hparamInfo.name, + type: hparamInfo.type, + domain, + }; +} + +function transformBackendMetricSpec( + metricInfo: backendTypes.MetricSpec +): MetricSpec { + const {name, ...otherSpec} = metricInfo; + return { + ...otherSpec, + tag: name.tag, + }; +} + +declare interface GetExperimentHparamRequestPayload { + experimentName: string; +} + @Injectable() export class TBRunsDataSource implements RunsDataSource { constructor(private readonly http: TBHttpClient) {} @@ -48,11 +98,112 @@ export class TBRunsDataSource implements RunsDataSource { } fetchHparamsMetadata(experimentId: string): Observable { - // Return a stub implementation. - return of({ - hparamSpecs: [], - metricSpecs: [], - runToHparamsAndMetrics: {}, - }); + const requestPayload: GetExperimentHparamRequestPayload = { + experimentName: experimentId, + }; + return this.http + .post( + `/experiment/${experimentId}/${HPARAMS_HTTP_PATH_PREFIX}/experiment`, + requestPayload + ) + .pipe( + map((response) => { + const colParams: backendTypes.BackendListSessionGroupRequest['colParams'] = + []; + + for (const hparamInfo of response.hparamInfos) { + colParams.push({hparam: hparamInfo.name}); + } + for (const metricInfo of response.metricInfos) { + colParams.push({metric: metricInfo.name}); + } + + const listSessionRequestParams: backendTypes.BackendListSessionGroupRequest = + { + experimentName: experimentId, + allowedStatuses: [ + backendTypes.RunStatus.STATUS_FAILURE, + backendTypes.RunStatus.STATUS_RUNNING, + backendTypes.RunStatus.STATUS_SUCCESS, + backendTypes.RunStatus.STATUS_UNKNOWN, + ], + colParams, + startIndex: 0, + // arbitrary large number so it does not get clipped. + sliceSize: 1e6, + }; + + return { + experimentHparamsInfo: response, + listSessionRequestParams, + }; + }), + mergeMap(({experimentHparamsInfo, listSessionRequestParams}) => { + return this.http + .post( + `/experiment/${experimentId}/${HPARAMS_HTTP_PATH_PREFIX}/session_groups`, + JSON.stringify(listSessionRequestParams) + ) + .pipe( + map((sessionGroupsList) => { + return {experimentHparamsInfo, sessionGroupsList}; + }) + ); + }), + map(({experimentHparamsInfo, sessionGroupsList}) => { + const runToHparamsAndMetrics: RunToHparamsAndMetrics = {}; + + // Reorganize the sessionGroup/session into run to . + for (const sessionGroup of sessionGroupsList.sessionGroups) { + const hparams: HparamValue[] = Object.entries( + sessionGroup.hparams + ).map((keyValue) => { + const [hparam, value] = keyValue; + return {name: hparam, value}; + }); + + for (const session of sessionGroup.sessions) { + for (const metricValue of session.metricValues) { + const runName = metricValue.name.group + ? `${session.name}/${metricValue.name.group}` + : session.name; + const runId = `${experimentId}/${runName}`; + const hparamsAndMetrics = runToHparamsAndMetrics[runId] || { + metrics: [], + hparams, + }; + hparamsAndMetrics.metrics.push({ + tag: metricValue.name.tag, + trainingStep: metricValue.trainingStep, + value: metricValue.value, + }); + runToHparamsAndMetrics[runId] = hparamsAndMetrics; + } + } + } + return { + hparamSpecs: experimentHparamsInfo.hparamInfos.map( + transformBackendHparamSpec + ), + metricSpecs: experimentHparamsInfo.metricInfos.map( + transformBackendMetricSpec + ), + runToHparamsAndMetrics, + }; + }), + catchError((error) => { + // HParams plugin return 400 when there are no hparams for an + // experiment. + if (error instanceof HttpErrorResponse && error.status === 400) { + return of({ + hparamSpecs: [], + metricSpecs: [], + runToHparamsAndMetrics: {}, + }); + } + return throwError(error); + }) + ); } } diff --git a/tensorboard/webapp/runs/data_source/runs_data_source_test.ts b/tensorboard/webapp/runs/data_source/runs_data_source_test.ts index bba145b027..64f509f863 100644 --- a/tensorboard/webapp/runs/data_source/runs_data_source_test.ts +++ b/tensorboard/webapp/runs/data_source/runs_data_source_test.ts @@ -18,7 +18,14 @@ import { TBHttpClientTestingModule, } from '../../webapp_data_source/tb_http_client_testing'; import {TBRunsDataSource} from './runs_data_source'; -import {RunsDataSource} from './runs_data_source_types'; +import {DomainType, RunsDataSource} from './runs_data_source_types'; +import { + createHparamsExperimentNoDomainResponse, + createHparamsExperimentResponse, + createHparamsListSessionGroupResponse, +} from './testing'; + +import * as types from './runs_backend_types'; describe('TBRunsDataSource test', () => { let httpMock: HttpTestingController; @@ -49,4 +56,271 @@ describe('TBRunsDataSource test', () => { ]); })); }); + + describe('#fetchHparamsMetadata', () => { + it( + 'calls /experiment and /session_groups to return map of run to ' + + 'hparams and metrics', + () => { + const returnValue = jasmine.createSpy(); + dataSource.fetchHparamsMetadata('eid').subscribe(returnValue); + httpMock + .expectOne('/experiment/eid/data/plugin/hparams/experiment') + .flush(createHparamsExperimentResponse()); + httpMock + .expectOne('/experiment/eid/data/plugin/hparams/session_groups') + .flush(createHparamsListSessionGroupResponse()); + + expect(returnValue).toHaveBeenCalledWith({ + hparamSpecs: [ + { + description: 'describes hparams one', + displayName: 'hparams one', + name: 'hparams1', + type: types.BackendHparamsValueType.DATA_TYPE_STRING, + domain: { + type: DomainType.INTERVAL, + minValue: -100, + maxValue: 100, + }, + }, + { + description: 'describes hparams two', + displayName: 'hparams two', + name: 'hparams2', + type: types.BackendHparamsValueType.DATA_TYPE_BOOL, + domain: { + type: DomainType.DISCRETE, + values: ['foo', 'bar', 'baz'], + }, + }, + ], + metricSpecs: [ + { + tag: 'metrics1', + displayName: 'Metrics One', + description: 'describe metrics one', + datasetType: types.DatasetType.DATASET_UNKNOWN, + }, + { + tag: 'metrics2', + displayName: 'Metrics Two', + description: 'describe metrics two', + datasetType: types.DatasetType.DATASET_TRAINING, + }, + ], + runToHparamsAndMetrics: { + 'eid/run_name_1': { + hparams: [ + {name: 'hparams1', value: -100}, + {name: 'hparams2', value: 'bar'}, + ], + metrics: [ + { + tag: 'metrics1', + trainingStep: 1000, + value: 1, + }, + ], + }, + 'eid/run_name_2/test': { + hparams: [ + {name: 'hparams1', value: 100}, + {name: 'hparams2', value: 'foo'}, + ], + metrics: [ + { + tag: 'metrics1', + trainingStep: 5000, + value: 0.6, + }, + ], + }, + 'eid/run_name_2/train': { + hparams: [ + {name: 'hparams1', value: 100}, + {name: 'hparams2', value: 'foo'}, + ], + metrics: [ + { + tag: 'metrics1', + trainingStep: 2000, + value: 0.1, + }, + { + tag: 'metrics1', + trainingStep: 10000, + value: 0.3, + }, + { + tag: 'metrics2', + trainingStep: 10000, + value: 0, + }, + ], + }, + }, + }); + } + ); + + it( + 'calls /experiment and /session_groups to return map of run to ' + + 'hparams and metrics with missing domain ranges', + () => { + const returnValue = jasmine.createSpy(); + dataSource.fetchHparamsMetadata('eid').subscribe(returnValue); + httpMock + .expectOne('/experiment/eid/data/plugin/hparams/experiment') + .flush(createHparamsExperimentNoDomainResponse()); + httpMock + .expectOne('/experiment/eid/data/plugin/hparams/session_groups') + .flush(createHparamsListSessionGroupResponse()); + + expect(returnValue).toHaveBeenCalledWith({ + hparamSpecs: [ + { + description: 'describes hparams one', + displayName: 'hparams one', + name: 'hparams1', + type: types.BackendHparamsValueType.DATA_TYPE_STRING, + domain: { + type: DomainType.INTERVAL, + minValue: -Infinity, + maxValue: Infinity, + }, + }, + { + description: 'describes hparams two', + displayName: 'hparams two', + name: 'hparams2', + type: types.BackendHparamsValueType.DATA_TYPE_BOOL, + domain: { + type: DomainType.DISCRETE, + values: ['foo', 'bar', 'baz'], + }, + }, + ], + metricSpecs: [ + { + tag: 'metrics1', + displayName: 'Metrics One', + description: 'describe metrics one', + datasetType: types.DatasetType.DATASET_UNKNOWN, + }, + { + tag: 'metrics2', + displayName: 'Metrics Two', + description: 'describe metrics two', + datasetType: types.DatasetType.DATASET_TRAINING, + }, + ], + runToHparamsAndMetrics: { + 'eid/run_name_1': { + hparams: [ + {name: 'hparams1', value: -100}, + {name: 'hparams2', value: 'bar'}, + ], + metrics: [ + { + tag: 'metrics1', + trainingStep: 1000, + value: 1, + }, + ], + }, + 'eid/run_name_2/test': { + hparams: [ + {name: 'hparams1', value: 100}, + {name: 'hparams2', value: 'foo'}, + ], + metrics: [ + { + tag: 'metrics1', + trainingStep: 5000, + value: 0.6, + }, + ], + }, + 'eid/run_name_2/train': { + hparams: [ + {name: 'hparams1', value: 100}, + {name: 'hparams2', value: 'foo'}, + ], + metrics: [ + { + tag: 'metrics1', + trainingStep: 2000, + value: 0.1, + }, + { + tag: 'metrics1', + trainingStep: 10000, + value: 0.3, + }, + { + tag: 'metrics2', + trainingStep: 10000, + value: 0, + }, + ], + }, + }, + }); + } + ); + + it('does not break when responses is empty', () => { + const returnValue = jasmine.createSpy(); + dataSource.fetchHparamsMetadata('eid').subscribe(returnValue); + httpMock + .expectOne('/experiment/eid/data/plugin/hparams/experiment') + .flush({ + description: '', + hparamInfos: [], + metricInfos: [], + name: '', + timeCreatedSecs: 0, + user: '', + }); + httpMock + .expectOne('/experiment/eid/data/plugin/hparams/session_groups') + .flush({ + sessionGroups: [], + totalSize: 0, + }); + + expect(returnValue).toHaveBeenCalledWith({ + hparamSpecs: [], + metricSpecs: [], + runToHparamsAndMetrics: {}, + }); + }); + + it('returns empty hparams when backend responds with 400', () => { + const returnValue = jasmine.createSpy(); + dataSource.fetchHparamsMetadata('eid').subscribe(returnValue); + httpMock + .expectOne('/experiment/eid/data/plugin/hparams/experiment') + .error(new ErrorEvent('400 error'), {status: 400}); + + expect(returnValue).toHaveBeenCalledWith({ + hparamSpecs: [], + metricSpecs: [], + runToHparamsAndMetrics: {}, + }); + }); + + it('throws error when response is 404', () => { + const returnValue = jasmine.createSpy(); + const errorValue = jasmine.createSpy(); + dataSource.fetchHparamsMetadata('eid').subscribe(returnValue, errorValue); + httpMock + .expectOne('/experiment/eid/data/plugin/hparams/experiment') + .error(new ErrorEvent('404 error'), {status: 404}); + + expect(returnValue).not.toHaveBeenCalled(); + expect(errorValue).toHaveBeenCalled(); + }); + }); }); diff --git a/tensorboard/webapp/runs/data_source/testing.ts b/tensorboard/webapp/runs/data_source/testing.ts index 2b7c3253bb..e47b0b95f2 100644 --- a/tensorboard/webapp/runs/data_source/testing.ts +++ b/tensorboard/webapp/runs/data_source/testing.ts @@ -14,7 +14,14 @@ limitations under the License. ==============================================================================*/ import {Injectable} from '@angular/core'; import {Observable, of} from 'rxjs'; -import {BackendHparamsValueType, DatasetType} from './runs_backend_types'; +import { + BackendHparamsExperimentResponse, + BackendHparamsValueType, + BackendListSessionGroupResponse, + DatasetType, + HparamSpec, + RunStatus, +} from './runs_backend_types'; import { DomainType, HparamsAndMetadata, @@ -69,3 +76,193 @@ export function provideTestingRunsDataSource() { {provide: RunsDataSource, useExisting: TestingRunsDataSource}, ]; } + +export function createHparamsExperimentResponse(): BackendHparamsExperimentResponse { + return { + description: 'some description', + hparamInfos: [ + { + description: 'describes hparams one', + displayName: 'hparams one', + name: 'hparams1', + type: BackendHparamsValueType.DATA_TYPE_STRING, + domainInterval: {minValue: -100, maxValue: 100}, + }, + { + description: 'describes hparams two', + displayName: 'hparams two', + name: 'hparams2', + type: BackendHparamsValueType.DATA_TYPE_BOOL, + domainDiscrete: ['foo', 'bar', 'baz'], + }, + ], + metricInfos: [ + { + name: { + group: '', + tag: 'metrics1', + }, + displayName: 'Metrics One', + description: 'describe metrics one', + datasetType: DatasetType.DATASET_UNKNOWN, + }, + { + name: { + group: 'group', + tag: 'metrics2', + }, + displayName: 'Metrics Two', + description: 'describe metrics two', + datasetType: DatasetType.DATASET_TRAINING, + }, + ], + name: 'experiment name', + timeCreatedSecs: 1337, + user: 'user name', + }; +} + +export function createHparamsExperimentNoDomainResponse(): BackendHparamsExperimentResponse { + return { + description: 'some description', + hparamInfos: [ + { + description: 'describes hparams one', + displayName: 'hparams one', + name: 'hparams1', + type: BackendHparamsValueType.DATA_TYPE_STRING, + } as HparamSpec, + { + description: 'describes hparams two', + displayName: 'hparams two', + name: 'hparams2', + type: BackendHparamsValueType.DATA_TYPE_BOOL, + domainDiscrete: ['foo', 'bar', 'baz'], + }, + ], + metricInfos: [ + { + name: { + group: '', + tag: 'metrics1', + }, + displayName: 'Metrics One', + description: 'describe metrics one', + datasetType: DatasetType.DATASET_UNKNOWN, + }, + { + name: { + group: 'group', + tag: 'metrics2', + }, + displayName: 'Metrics Two', + description: 'describe metrics two', + datasetType: DatasetType.DATASET_TRAINING, + }, + ], + name: 'experiment name', + timeCreatedSecs: 1337, + user: 'user name', + }; +} + +export function createHparamsListSessionGroupResponse(): BackendListSessionGroupResponse { + return { + sessionGroups: [ + { + name: 'session_id_1', + hparams: { + hparams1: -100, + hparams2: 'bar', + }, + sessions: [ + { + endTimeSecs: 0, + metricValues: [ + { + name: { + group: '', + tag: 'metrics1', + }, + trainingStep: 1000, + value: 1, + wallTimeSecs: 0, + }, + ], + modelUri: '', + monitorUrl: '', + name: 'run_name_1', + startTimeSecs: 0, + status: RunStatus.STATUS_SUCCESS, + }, + ], + }, + { + name: 'session_id_2', + hparams: { + hparams1: 100, + hparams2: 'foo', + }, + sessions: [ + { + endTimeSecs: 0, + metricValues: [ + { + name: { + group: 'train', + tag: 'metrics1', + }, + trainingStep: 2000, + value: 0.1, + wallTimeSecs: 0, + }, + { + name: { + group: 'test', + tag: 'metrics1', + }, + trainingStep: 5000, + value: 0.6, + wallTimeSecs: 0, + }, + ], + modelUri: '', + monitorUrl: '', + name: 'run_name_2', + startTimeSecs: 0, + status: RunStatus.STATUS_SUCCESS, + }, + { + endTimeSecs: 0, + metricValues: [ + { + name: { + group: 'train', + tag: 'metrics1', + }, + trainingStep: 10000, + value: 0.3, + wallTimeSecs: 0, + }, + { + name: { + group: 'train', + tag: 'metrics2', + }, + trainingStep: 10000, + value: 0, + wallTimeSecs: 0, + }, + ], + modelUri: '', + monitorUrl: '', + name: 'run_name_2', + startTimeSecs: 0, + status: RunStatus.STATUS_RUNNING, + }, + ], + }, + ], + totalSize: 2, + }; +}