diff --git a/src/nni_manager/common/datastore.ts b/src/nni_manager/common/datastore.ts index 7ed2328d7d..d3cca0479b 100644 --- a/src/nni_manager/common/datastore.ts +++ b/src/nni_manager/common/datastore.ts @@ -23,7 +23,7 @@ import { ExperimentProfile, TrialJobStatistics } from './manager'; import { TrialJobDetail, TrialJobStatus } from './trainingService'; type TrialJobEvent = TrialJobStatus | 'USER_TO_CANCEL' | 'ADD_CUSTOMIZED' | 'ADD_HYPERPARAMETER'; -type MetricType = 'PERIODICAL' | 'FINAL' | 'CUSTOM'; +type MetricType = 'PERIODICAL' | 'FINAL' | 'CUSTOM' | 'REQUEST_PARAMETER'; interface ExperimentProfileRecord { readonly timestamp: number; diff --git a/src/nni_manager/core/nniDataStore.ts b/src/nni_manager/core/nniDataStore.ts index 1beec632be..f03c9a81f1 100644 --- a/src/nni_manager/core/nniDataStore.ts +++ b/src/nni_manager/core/nniDataStore.ts @@ -26,6 +26,7 @@ import * as component from '../common/component'; import { Database, DataStore, MetricData, MetricDataRecord, MetricType, TrialJobEvent, TrialJobEventRecord, TrialJobInfo } from '../common/datastore'; import { isNewExperiment } from '../common/experimentStartupInfo'; +import { getExperimentId } from '../common/experimentStartupInfo'; import { getLogger, Logger } from '../common/log'; import { ExperimentProfile, TrialJobStatistics } from '../common/manager'; import { TrialJobStatus } from '../common/trainingService'; @@ -35,6 +36,7 @@ class NNIDataStore implements DataStore { private db: Database = component.get(Database); private log: Logger = getLogger(); private initTask!: Deferred; + private multiPhase: boolean | undefined; public init(): Promise { if (this.initTask !== undefined) { @@ -112,14 +114,18 @@ class NNIDataStore implements DataStore { } public async getTrialJob(trialJobId: string): Promise { - const trialJobs = await this.queryTrialJobs(undefined, trialJobId); + const trialJobs: TrialJobInfo[] = await this.queryTrialJobs(undefined, trialJobId); return trialJobs[0]; } public async storeMetricData(trialJobId: string, data: string): Promise { this.log.debug(`storeMetricData: trialJobId: ${trialJobId}, data: ${data}`); - const metrics = JSON.parse(data) as MetricData; + const metrics: MetricData = JSON.parse(data); + if (metrics.type === 'REQUEST_PARAMETER') { + + return; + } assert(trialJobId === metrics.trial_job_id); await this.db.storeMetricData(trialJobId, JSON.stringify({ trialJobId: metrics.trial_job_id, @@ -161,11 +167,25 @@ class NNIDataStore implements DataStore { private async getFinalMetricData(trialJobId: string): Promise { const metrics: MetricDataRecord[] = await this.getMetricData(trialJobId, 'FINAL'); - assert(metrics.length <= 1); - if (metrics.length === 1) { - return metrics[0]; + + const multiPhase: boolean | undefined = await this.isMultiPhase(); + + if (metrics.length > 1 && !multiPhase) { + this.log.error(`Found multiple FINAL results for trial job ${trialJobId}`); + } + + return metrics[metrics.length - 1]; + } + + private async isMultiPhase(): Promise { + if (this.multiPhase === undefined) { + this.multiPhase = (await this.getExperimentProfile(getExperimentId())).params.multiPhase; + } + + if (this.multiPhase !== undefined) { + return this.multiPhase; } else { - return undefined; + return false; } }