From 4e32569f870eb95be23293c958ee3c1aff57c7ab Mon Sep 17 00:00:00 2001 From: Chengmin Chi Date: Wed, 26 Sep 2018 12:15:17 +0800 Subject: [PATCH 1/5] Fix multiphase datastore problem --- src/nni_manager/common/datastore.ts | 2 +- src/nni_manager/core/nniDataStore.ts | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/src/nni_manager/common/datastore.ts b/src/nni_manager/common/datastore.ts index 7ed2328d7d..d3cca0479b 100644 --- a/src/nni_manager/common/datastore.ts +++ b/src/nni_manager/common/datastore.ts @@ -23,7 +23,7 @@ import { ExperimentProfile, TrialJobStatistics } from './manager'; import { TrialJobDetail, TrialJobStatus } from './trainingService'; type TrialJobEvent = TrialJobStatus | 'USER_TO_CANCEL' | 'ADD_CUSTOMIZED' | 'ADD_HYPERPARAMETER'; -type MetricType = 'PERIODICAL' | 'FINAL' | 'CUSTOM'; +type MetricType = 'PERIODICAL' | 'FINAL' | 'CUSTOM' | 'REQUEST_PARAMETER'; interface ExperimentProfileRecord { readonly timestamp: number; diff --git a/src/nni_manager/core/nniDataStore.ts b/src/nni_manager/core/nniDataStore.ts index 1beec632be..fc7edbdc6f 100644 --- a/src/nni_manager/core/nniDataStore.ts +++ b/src/nni_manager/core/nniDataStore.ts @@ -112,14 +112,18 @@ class NNIDataStore implements DataStore { } public async getTrialJob(trialJobId: string): Promise { - const trialJobs = await this.queryTrialJobs(undefined, trialJobId); + const trialJobs: TrialJobInfo[] = await this.queryTrialJobs(undefined, trialJobId); return trialJobs[0]; } public async storeMetricData(trialJobId: string, data: string): Promise { this.log.debug(`storeMetricData: trialJobId: ${trialJobId}, data: ${data}`); - const metrics = JSON.parse(data) as MetricData; + const metrics: MetricData = JSON.parse(data); + if (metrics.type === 'REQUEST_PARAMETER') { + + return; + } assert(trialJobId === metrics.trial_job_id); await this.db.storeMetricData(trialJobId, JSON.stringify({ trialJobId: metrics.trial_job_id, @@ -161,9 +165,8 @@ class NNIDataStore implements DataStore { private async getFinalMetricData(trialJobId: string): Promise { const metrics: MetricDataRecord[] = await this.getMetricData(trialJobId, 'FINAL'); - assert(metrics.length <= 1); - if (metrics.length === 1) { - return metrics[0]; + if (metrics.length > 0) { + return metrics[metrics.length - 1]; } else { return undefined; } From a1e30f1bc63747f99781d946bceb119ccfb00c7b Mon Sep 17 00:00:00 2001 From: Chengmin Chi Date: Wed, 26 Sep 2018 15:24:27 +0800 Subject: [PATCH 2/5] updates --- src/nni_manager/core/nniDataStore.ts | 29 +++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/src/nni_manager/core/nniDataStore.ts b/src/nni_manager/core/nniDataStore.ts index fc7edbdc6f..88b39f830b 100644 --- a/src/nni_manager/core/nniDataStore.ts +++ b/src/nni_manager/core/nniDataStore.ts @@ -26,6 +26,7 @@ import * as component from '../common/component'; import { Database, DataStore, MetricData, MetricDataRecord, MetricType, TrialJobEvent, TrialJobEventRecord, TrialJobInfo } from '../common/datastore'; import { isNewExperiment } from '../common/experimentStartupInfo'; +import { getExperimentId } from '../common/experimentStartupInfo'; import { getLogger, Logger } from '../common/log'; import { ExperimentProfile, TrialJobStatistics } from '../common/manager'; import { TrialJobStatus } from '../common/trainingService'; @@ -35,6 +36,7 @@ class NNIDataStore implements DataStore { private db: Database = component.get(Database); private log: Logger = getLogger(); private initTask!: Deferred; + private multiPhase: boolean | undefined; public init(): Promise { if (this.initTask !== undefined) { @@ -165,10 +167,31 @@ class NNIDataStore implements DataStore { private async getFinalMetricData(trialJobId: string): Promise { const metrics: MetricDataRecord[] = await this.getMetricData(trialJobId, 'FINAL'); - if (metrics.length > 0) { - return metrics[metrics.length - 1]; + + const multiPhase: boolean | undefined = await this.isMultiPhase(); + + if (multiPhase) { + if (metrics.length > 0) { + return metrics[metrics.length - 1]; + } else { + return undefined; + } + } else { + assert(metrics.length <= 1); + + return metrics[0]; + } + } + + private async isMultiPhase(): Promise { + if (this.multiPhase === undefined) { + this.multiPhase = (await this.getExperimentProfile(getExperimentId())).params.multiPhase; + } + + if (this.multiPhase !== undefined) { + return this.multiPhase; } else { - return undefined; + assert(false, 'Failed to retrieve multiPhase flag'); } } From bf08233e3af930f8e90cf8ced05402f4888e3adf Mon Sep 17 00:00:00 2001 From: Chengmin Chi Date: Wed, 26 Sep 2018 17:34:57 +0800 Subject: [PATCH 3/5] updates --- src/nni_manager/core/nniDataStore.ts | 16 +++++----------- 1 file changed, 5 insertions(+), 11 deletions(-) diff --git a/src/nni_manager/core/nniDataStore.ts b/src/nni_manager/core/nniDataStore.ts index 88b39f830b..db3adfdbdc 100644 --- a/src/nni_manager/core/nniDataStore.ts +++ b/src/nni_manager/core/nniDataStore.ts @@ -170,17 +170,11 @@ class NNIDataStore implements DataStore { const multiPhase: boolean | undefined = await this.isMultiPhase(); - if (multiPhase) { - if (metrics.length > 0) { - return metrics[metrics.length - 1]; - } else { - return undefined; - } - } else { - assert(metrics.length <= 1); - - return metrics[0]; + if (metrics.length > 1 && !multiPhase) { + this.log.error(`Found multiple FINAL results for trial job ${trialJobId}`); } + + return metrics[metrics.length - 1]; } private async isMultiPhase(): Promise { @@ -188,7 +182,7 @@ class NNIDataStore implements DataStore { this.multiPhase = (await this.getExperimentProfile(getExperimentId())).params.multiPhase; } - if (this.multiPhase !== undefined) { + if (this.multiPhase) { return this.multiPhase; } else { assert(false, 'Failed to retrieve multiPhase flag'); From fdc0c15b14fd3f1a916f2603c12b498e23eb7f68 Mon Sep 17 00:00:00 2001 From: Chengmin Chi Date: Wed, 26 Sep 2018 20:04:39 +0800 Subject: [PATCH 4/5] updates --- src/nni_manager/core/nniDataStore.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nni_manager/core/nniDataStore.ts b/src/nni_manager/core/nniDataStore.ts index db3adfdbdc..49d0a5f89b 100644 --- a/src/nni_manager/core/nniDataStore.ts +++ b/src/nni_manager/core/nniDataStore.ts @@ -182,7 +182,7 @@ class NNIDataStore implements DataStore { this.multiPhase = (await this.getExperimentProfile(getExperimentId())).params.multiPhase; } - if (this.multiPhase) { + if (this.multiPhase !== undefined) { return this.multiPhase; } else { assert(false, 'Failed to retrieve multiPhase flag'); From d832e62ddab95b024cdfb696fa665dda3ad2828c Mon Sep 17 00:00:00 2001 From: Chengmin Chi Date: Wed, 26 Sep 2018 20:07:09 +0800 Subject: [PATCH 5/5] updates --- src/nni_manager/core/nniDataStore.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nni_manager/core/nniDataStore.ts b/src/nni_manager/core/nniDataStore.ts index 49d0a5f89b..f03c9a81f1 100644 --- a/src/nni_manager/core/nniDataStore.ts +++ b/src/nni_manager/core/nniDataStore.ts @@ -185,7 +185,7 @@ class NNIDataStore implements DataStore { if (this.multiPhase !== undefined) { return this.multiPhase; } else { - assert(false, 'Failed to retrieve multiPhase flag'); + return false; } }