diff --git a/tensorboard/webapp/runs/data_source/BUILD b/tensorboard/webapp/runs/data_source/BUILD index b35f4e4705..ccf9384937 100644 --- a/tensorboard/webapp/runs/data_source/BUILD +++ b/tensorboard/webapp/runs/data_source/BUILD @@ -48,9 +48,7 @@ tf_ts_library( "runs_data_source_test.ts", ], deps = [ - ":backend_types", ":data_source", - ":testing", "//tensorboard/webapp/angular:expect_angular_core_testing", "//tensorboard/webapp/webapp_data_source:http_client_testing", "@npm//@types/jasmine", diff --git a/tensorboard/webapp/runs/data_source/runs_data_source.ts b/tensorboard/webapp/runs/data_source/runs_data_source.ts index 854caabc33..6c7c309d73 100644 --- a/tensorboard/webapp/runs/data_source/runs_data_source.ts +++ b/tensorboard/webapp/runs/data_source/runs_data_source.ts @@ -13,71 +13,21 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ import {Injectable} from '@angular/core'; -import {Observable, of, throwError} from 'rxjs'; -import {catchError, map, mergeMap} from 'rxjs/operators'; +import {Observable, of} from 'rxjs'; +import {map} from 'rxjs/operators'; +import {TBHttpClient} from '../../webapp_data_source/tb_http_client'; import { - HttpErrorResponse, - TBHttpClient, -} from '../../webapp_data_source/tb_http_client'; -import * as backendTypes from './runs_backend_types'; -import { - Domain, - DomainType, HparamsAndMetadata, - HparamSpec, - HparamValue, - MetricSpec, Run, RunsDataSource, - RunToHparamsAndMetrics, } from './runs_data_source_types'; -const HPARAMS_HTTP_PATH_PREFIX = 'data/plugin/hparams'; - type BackendGetRunsResponse = string[]; function runToRunId(run: string, experimentId: string) { return `${experimentId}/${run}`; } -function transformBackendHparamSpec( - hparamInfo: backendTypes.HparamSpec -): HparamSpec { - let domain: Domain; - if (backendTypes.isDiscreteDomainHparamSpec(hparamInfo)) { - domain = {type: DomainType.DISCRETE, values: hparamInfo.domainDiscrete}; - } else if (backendTypes.isIntervalDomainHparamSpec(hparamInfo)) { - domain = {...hparamInfo.domainInterval, type: DomainType.INTERVAL}; - } else { - domain = { - type: DomainType.INTERVAL, - minValue: -Infinity, - maxValue: Infinity, - }; - } - return { - description: hparamInfo.description, - displayName: hparamInfo.displayName, - name: hparamInfo.name, - type: hparamInfo.type, - domain, - }; -} - -function transformBackendMetricSpec( - metricInfo: backendTypes.MetricSpec -): MetricSpec { - const {name, ...otherSpec} = metricInfo; - return { - ...otherSpec, - tag: name.tag, - }; -} - -declare interface GetExperimentHparamRequestPayload { - experimentName: string; -} - @Injectable() export class TBRunsDataSource implements RunsDataSource { constructor(private readonly http: TBHttpClient) {} @@ -98,112 +48,11 @@ export class TBRunsDataSource implements RunsDataSource { } fetchHparamsMetadata(experimentId: string): Observable { - const requestPayload: GetExperimentHparamRequestPayload = { - experimentName: experimentId, - }; - return this.http - .post( - `/experiment/${experimentId}/${HPARAMS_HTTP_PATH_PREFIX}/experiment`, - requestPayload - ) - .pipe( - map((response) => { - const colParams: backendTypes.BackendListSessionGroupRequest['colParams'] = - []; - - for (const hparamInfo of response.hparamInfos) { - colParams.push({hparam: hparamInfo.name}); - } - for (const metricInfo of response.metricInfos) { - colParams.push({metric: metricInfo.name}); - } - - const listSessionRequestParams: backendTypes.BackendListSessionGroupRequest = - { - experimentName: experimentId, - allowedStatuses: [ - backendTypes.RunStatus.STATUS_FAILURE, - backendTypes.RunStatus.STATUS_RUNNING, - backendTypes.RunStatus.STATUS_SUCCESS, - backendTypes.RunStatus.STATUS_UNKNOWN, - ], - colParams, - startIndex: 0, - // arbitrary large number so it does not get clipped. - sliceSize: 1e6, - }; - - return { - experimentHparamsInfo: response, - listSessionRequestParams, - }; - }), - mergeMap(({experimentHparamsInfo, listSessionRequestParams}) => { - return this.http - .post( - `/experiment/${experimentId}/${HPARAMS_HTTP_PATH_PREFIX}/session_groups`, - JSON.stringify(listSessionRequestParams) - ) - .pipe( - map((sessionGroupsList) => { - return {experimentHparamsInfo, sessionGroupsList}; - }) - ); - }), - map(({experimentHparamsInfo, sessionGroupsList}) => { - const runToHparamsAndMetrics: RunToHparamsAndMetrics = {}; - - // Reorganize the sessionGroup/session into run to . - for (const sessionGroup of sessionGroupsList.sessionGroups) { - const hparams: HparamValue[] = Object.entries( - sessionGroup.hparams - ).map((keyValue) => { - const [hparam, value] = keyValue; - return {name: hparam, value}; - }); - - for (const session of sessionGroup.sessions) { - for (const metricValue of session.metricValues) { - const runName = metricValue.name.group - ? `${session.name}/${metricValue.name.group}` - : session.name; - const runId = `${experimentId}/${runName}`; - const hparamsAndMetrics = runToHparamsAndMetrics[runId] || { - metrics: [], - hparams, - }; - hparamsAndMetrics.metrics.push({ - tag: metricValue.name.tag, - trainingStep: metricValue.trainingStep, - value: metricValue.value, - }); - runToHparamsAndMetrics[runId] = hparamsAndMetrics; - } - } - } - return { - hparamSpecs: experimentHparamsInfo.hparamInfos.map( - transformBackendHparamSpec - ), - metricSpecs: experimentHparamsInfo.metricInfos.map( - transformBackendMetricSpec - ), - runToHparamsAndMetrics, - }; - }), - catchError((error) => { - // HParams plugin return 400 when there are no hparams for an - // experiment. - if (error instanceof HttpErrorResponse && error.status === 400) { - return of({ - hparamSpecs: [], - metricSpecs: [], - runToHparamsAndMetrics: {}, - }); - } - return throwError(error); - }) - ); + // Return a stub implementation. + return of({ + hparamSpecs: [], + metricSpecs: [], + runToHparamsAndMetrics: {}, + }); } } diff --git a/tensorboard/webapp/runs/data_source/runs_data_source_test.ts b/tensorboard/webapp/runs/data_source/runs_data_source_test.ts index 64f509f863..bba145b027 100644 --- a/tensorboard/webapp/runs/data_source/runs_data_source_test.ts +++ b/tensorboard/webapp/runs/data_source/runs_data_source_test.ts @@ -18,14 +18,7 @@ import { TBHttpClientTestingModule, } from '../../webapp_data_source/tb_http_client_testing'; import {TBRunsDataSource} from './runs_data_source'; -import {DomainType, RunsDataSource} from './runs_data_source_types'; -import { - createHparamsExperimentNoDomainResponse, - createHparamsExperimentResponse, - createHparamsListSessionGroupResponse, -} from './testing'; - -import * as types from './runs_backend_types'; +import {RunsDataSource} from './runs_data_source_types'; describe('TBRunsDataSource test', () => { let httpMock: HttpTestingController; @@ -56,271 +49,4 @@ describe('TBRunsDataSource test', () => { ]); })); }); - - describe('#fetchHparamsMetadata', () => { - it( - 'calls /experiment and /session_groups to return map of run to ' + - 'hparams and metrics', - () => { - const returnValue = jasmine.createSpy(); - dataSource.fetchHparamsMetadata('eid').subscribe(returnValue); - httpMock - .expectOne('/experiment/eid/data/plugin/hparams/experiment') - .flush(createHparamsExperimentResponse()); - httpMock - .expectOne('/experiment/eid/data/plugin/hparams/session_groups') - .flush(createHparamsListSessionGroupResponse()); - - expect(returnValue).toHaveBeenCalledWith({ - hparamSpecs: [ - { - description: 'describes hparams one', - displayName: 'hparams one', - name: 'hparams1', - type: types.BackendHparamsValueType.DATA_TYPE_STRING, - domain: { - type: DomainType.INTERVAL, - minValue: -100, - maxValue: 100, - }, - }, - { - description: 'describes hparams two', - displayName: 'hparams two', - name: 'hparams2', - type: types.BackendHparamsValueType.DATA_TYPE_BOOL, - domain: { - type: DomainType.DISCRETE, - values: ['foo', 'bar', 'baz'], - }, - }, - ], - metricSpecs: [ - { - tag: 'metrics1', - displayName: 'Metrics One', - description: 'describe metrics one', - datasetType: types.DatasetType.DATASET_UNKNOWN, - }, - { - tag: 'metrics2', - displayName: 'Metrics Two', - description: 'describe metrics two', - datasetType: types.DatasetType.DATASET_TRAINING, - }, - ], - runToHparamsAndMetrics: { - 'eid/run_name_1': { - hparams: [ - {name: 'hparams1', value: -100}, - {name: 'hparams2', value: 'bar'}, - ], - metrics: [ - { - tag: 'metrics1', - trainingStep: 1000, - value: 1, - }, - ], - }, - 'eid/run_name_2/test': { - hparams: [ - {name: 'hparams1', value: 100}, - {name: 'hparams2', value: 'foo'}, - ], - metrics: [ - { - tag: 'metrics1', - trainingStep: 5000, - value: 0.6, - }, - ], - }, - 'eid/run_name_2/train': { - hparams: [ - {name: 'hparams1', value: 100}, - {name: 'hparams2', value: 'foo'}, - ], - metrics: [ - { - tag: 'metrics1', - trainingStep: 2000, - value: 0.1, - }, - { - tag: 'metrics1', - trainingStep: 10000, - value: 0.3, - }, - { - tag: 'metrics2', - trainingStep: 10000, - value: 0, - }, - ], - }, - }, - }); - } - ); - - it( - 'calls /experiment and /session_groups to return map of run to ' + - 'hparams and metrics with missing domain ranges', - () => { - const returnValue = jasmine.createSpy(); - dataSource.fetchHparamsMetadata('eid').subscribe(returnValue); - httpMock - .expectOne('/experiment/eid/data/plugin/hparams/experiment') - .flush(createHparamsExperimentNoDomainResponse()); - httpMock - .expectOne('/experiment/eid/data/plugin/hparams/session_groups') - .flush(createHparamsListSessionGroupResponse()); - - expect(returnValue).toHaveBeenCalledWith({ - hparamSpecs: [ - { - description: 'describes hparams one', - displayName: 'hparams one', - name: 'hparams1', - type: types.BackendHparamsValueType.DATA_TYPE_STRING, - domain: { - type: DomainType.INTERVAL, - minValue: -Infinity, - maxValue: Infinity, - }, - }, - { - description: 'describes hparams two', - displayName: 'hparams two', - name: 'hparams2', - type: types.BackendHparamsValueType.DATA_TYPE_BOOL, - domain: { - type: DomainType.DISCRETE, - values: ['foo', 'bar', 'baz'], - }, - }, - ], - metricSpecs: [ - { - tag: 'metrics1', - displayName: 'Metrics One', - description: 'describe metrics one', - datasetType: types.DatasetType.DATASET_UNKNOWN, - }, - { - tag: 'metrics2', - displayName: 'Metrics Two', - description: 'describe metrics two', - datasetType: types.DatasetType.DATASET_TRAINING, - }, - ], - runToHparamsAndMetrics: { - 'eid/run_name_1': { - hparams: [ - {name: 'hparams1', value: -100}, - {name: 'hparams2', value: 'bar'}, - ], - metrics: [ - { - tag: 'metrics1', - trainingStep: 1000, - value: 1, - }, - ], - }, - 'eid/run_name_2/test': { - hparams: [ - {name: 'hparams1', value: 100}, - {name: 'hparams2', value: 'foo'}, - ], - metrics: [ - { - tag: 'metrics1', - trainingStep: 5000, - value: 0.6, - }, - ], - }, - 'eid/run_name_2/train': { - hparams: [ - {name: 'hparams1', value: 100}, - {name: 'hparams2', value: 'foo'}, - ], - metrics: [ - { - tag: 'metrics1', - trainingStep: 2000, - value: 0.1, - }, - { - tag: 'metrics1', - trainingStep: 10000, - value: 0.3, - }, - { - tag: 'metrics2', - trainingStep: 10000, - value: 0, - }, - ], - }, - }, - }); - } - ); - - it('does not break when responses is empty', () => { - const returnValue = jasmine.createSpy(); - dataSource.fetchHparamsMetadata('eid').subscribe(returnValue); - httpMock - .expectOne('/experiment/eid/data/plugin/hparams/experiment') - .flush({ - description: '', - hparamInfos: [], - metricInfos: [], - name: '', - timeCreatedSecs: 0, - user: '', - }); - httpMock - .expectOne('/experiment/eid/data/plugin/hparams/session_groups') - .flush({ - sessionGroups: [], - totalSize: 0, - }); - - expect(returnValue).toHaveBeenCalledWith({ - hparamSpecs: [], - metricSpecs: [], - runToHparamsAndMetrics: {}, - }); - }); - - it('returns empty hparams when backend responds with 400', () => { - const returnValue = jasmine.createSpy(); - dataSource.fetchHparamsMetadata('eid').subscribe(returnValue); - httpMock - .expectOne('/experiment/eid/data/plugin/hparams/experiment') - .error(new ErrorEvent('400 error'), {status: 400}); - - expect(returnValue).toHaveBeenCalledWith({ - hparamSpecs: [], - metricSpecs: [], - runToHparamsAndMetrics: {}, - }); - }); - - it('throws error when response is 404', () => { - const returnValue = jasmine.createSpy(); - const errorValue = jasmine.createSpy(); - dataSource.fetchHparamsMetadata('eid').subscribe(returnValue, errorValue); - httpMock - .expectOne('/experiment/eid/data/plugin/hparams/experiment') - .error(new ErrorEvent('404 error'), {status: 404}); - - expect(returnValue).not.toHaveBeenCalled(); - expect(errorValue).toHaveBeenCalled(); - }); - }); }); diff --git a/tensorboard/webapp/runs/data_source/testing.ts b/tensorboard/webapp/runs/data_source/testing.ts index e47b0b95f2..2b7c3253bb 100644 --- a/tensorboard/webapp/runs/data_source/testing.ts +++ b/tensorboard/webapp/runs/data_source/testing.ts @@ -14,14 +14,7 @@ limitations under the License. ==============================================================================*/ import {Injectable} from '@angular/core'; import {Observable, of} from 'rxjs'; -import { - BackendHparamsExperimentResponse, - BackendHparamsValueType, - BackendListSessionGroupResponse, - DatasetType, - HparamSpec, - RunStatus, -} from './runs_backend_types'; +import {BackendHparamsValueType, DatasetType} from './runs_backend_types'; import { DomainType, HparamsAndMetadata, @@ -76,193 +69,3 @@ export function provideTestingRunsDataSource() { {provide: RunsDataSource, useExisting: TestingRunsDataSource}, ]; } - -export function createHparamsExperimentResponse(): BackendHparamsExperimentResponse { - return { - description: 'some description', - hparamInfos: [ - { - description: 'describes hparams one', - displayName: 'hparams one', - name: 'hparams1', - type: BackendHparamsValueType.DATA_TYPE_STRING, - domainInterval: {minValue: -100, maxValue: 100}, - }, - { - description: 'describes hparams two', - displayName: 'hparams two', - name: 'hparams2', - type: BackendHparamsValueType.DATA_TYPE_BOOL, - domainDiscrete: ['foo', 'bar', 'baz'], - }, - ], - metricInfos: [ - { - name: { - group: '', - tag: 'metrics1', - }, - displayName: 'Metrics One', - description: 'describe metrics one', - datasetType: DatasetType.DATASET_UNKNOWN, - }, - { - name: { - group: 'group', - tag: 'metrics2', - }, - displayName: 'Metrics Two', - description: 'describe metrics two', - datasetType: DatasetType.DATASET_TRAINING, - }, - ], - name: 'experiment name', - timeCreatedSecs: 1337, - user: 'user name', - }; -} - -export function createHparamsExperimentNoDomainResponse(): BackendHparamsExperimentResponse { - return { - description: 'some description', - hparamInfos: [ - { - description: 'describes hparams one', - displayName: 'hparams one', - name: 'hparams1', - type: BackendHparamsValueType.DATA_TYPE_STRING, - } as HparamSpec, - { - description: 'describes hparams two', - displayName: 'hparams two', - name: 'hparams2', - type: BackendHparamsValueType.DATA_TYPE_BOOL, - domainDiscrete: ['foo', 'bar', 'baz'], - }, - ], - metricInfos: [ - { - name: { - group: '', - tag: 'metrics1', - }, - displayName: 'Metrics One', - description: 'describe metrics one', - datasetType: DatasetType.DATASET_UNKNOWN, - }, - { - name: { - group: 'group', - tag: 'metrics2', - }, - displayName: 'Metrics Two', - description: 'describe metrics two', - datasetType: DatasetType.DATASET_TRAINING, - }, - ], - name: 'experiment name', - timeCreatedSecs: 1337, - user: 'user name', - }; -} - -export function createHparamsListSessionGroupResponse(): BackendListSessionGroupResponse { - return { - sessionGroups: [ - { - name: 'session_id_1', - hparams: { - hparams1: -100, - hparams2: 'bar', - }, - sessions: [ - { - endTimeSecs: 0, - metricValues: [ - { - name: { - group: '', - tag: 'metrics1', - }, - trainingStep: 1000, - value: 1, - wallTimeSecs: 0, - }, - ], - modelUri: '', - monitorUrl: '', - name: 'run_name_1', - startTimeSecs: 0, - status: RunStatus.STATUS_SUCCESS, - }, - ], - }, - { - name: 'session_id_2', - hparams: { - hparams1: 100, - hparams2: 'foo', - }, - sessions: [ - { - endTimeSecs: 0, - metricValues: [ - { - name: { - group: 'train', - tag: 'metrics1', - }, - trainingStep: 2000, - value: 0.1, - wallTimeSecs: 0, - }, - { - name: { - group: 'test', - tag: 'metrics1', - }, - trainingStep: 5000, - value: 0.6, - wallTimeSecs: 0, - }, - ], - modelUri: '', - monitorUrl: '', - name: 'run_name_2', - startTimeSecs: 0, - status: RunStatus.STATUS_SUCCESS, - }, - { - endTimeSecs: 0, - metricValues: [ - { - name: { - group: 'train', - tag: 'metrics1', - }, - trainingStep: 10000, - value: 0.3, - wallTimeSecs: 0, - }, - { - name: { - group: 'train', - tag: 'metrics2', - }, - trainingStep: 10000, - value: 0, - wallTimeSecs: 0, - }, - ], - modelUri: '', - monitorUrl: '', - name: 'run_name_2', - startTimeSecs: 0, - status: RunStatus.STATUS_RUNNING, - }, - ], - }, - ], - totalSize: 2, - }; -}