From 7242098efdd77d4bdfbb69c5c14ab6d705a8daf7 Mon Sep 17 00:00:00 2001 From: ShufanHuang Date: Mon, 25 Feb 2019 19:58:19 +0800 Subject: [PATCH] Improve the performance of the curve fitting assessor (#717) Improve the performance of the curve fitting assessor --- docs/en_US/Builtin_Assessors.md | 2 ++ docs/en_US/curvefittingAssessor.md | 4 ++++ .../curvefitting_assessor.py | 20 +++++++++++++++++-- .../curvefitting_assessor/model_factory.py | 12 +++++------ tools/nni_cmd/config_schema.py | 3 ++- 5 files changed, 32 insertions(+), 9 deletions(-) diff --git a/docs/en_US/Builtin_Assessors.md b/docs/en_US/Builtin_Assessors.md index 479be0da9a..72684763fe 100644 --- a/docs/en_US/Builtin_Assessors.md +++ b/docs/en_US/Builtin_Assessors.md @@ -63,6 +63,7 @@ It is applicable in a wide range of performance curves, thus, can be used in var * **optimize_mode** (*maximize or minimize, optional, default = maximize*) - If 'maximize', assessor will **stop** the trial with smaller expectation. If 'minimize', assessor will **stop** the trial with larger expectation. * **start_step** (*int, optional, default = 6*) - A trial is determined to be stopped or not, we start to predict only after receiving start_step number of reported intermediate results. * **threshold** (*float, optional, default = 0.95*) - The threshold that we decide to early stop the worse performance curve. For example: if threshold = 0.95, optimize_mode = maximize, best performance in the history is 0.9, then we will stop the trial which predict value is lower than 0.95 * 0.9 = 0.855. +* **gap** (*int, optional, default = 1*) - The gap interval between Assesor judgements. For example: if gap = 2, start_step = 6, then we will assess the result when we get 6, 8, 10, 12...intermedian result. **Usage example:** @@ -75,4 +76,5 @@ assessor: optimize_mode: maximize start_step: 6 threshold: 0.95 + gap: 1 ``` \ No newline at end of file diff --git a/docs/en_US/curvefittingAssessor.md b/docs/en_US/curvefittingAssessor.md index a4658c7cda..2be3bc2bcb 100644 --- a/docs/en_US/curvefittingAssessor.md +++ b/docs/en_US/curvefittingAssessor.md @@ -57,6 +57,10 @@ assessor: * The default value of threshold is 0.95. # Kindly reminds that if you choose minimize mode, please adjust the value of threshold >= 1.0 (e.g threshold=1.1) threshold: 0.95 + # (optional) The gap interval between Assesor judgements. + # For example: if gap = 2, start_step = 6, then we will assess the result when we get 6, 8, 10, 12...intermedian result. + * The default value of gap is 1. + gap: 1 ``` ## 3. File Structure diff --git a/src/sdk/pynni/nni/curvefitting_assessor/curvefitting_assessor.py b/src/sdk/pynni/nni/curvefitting_assessor/curvefitting_assessor.py index 012e370ba9..8d13dd6b1d 100644 --- a/src/sdk/pynni/nni/curvefitting_assessor/curvefitting_assessor.py +++ b/src/sdk/pynni/nni/curvefitting_assessor/curvefitting_assessor.py @@ -16,6 +16,7 @@ # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. import logging +import datetime from nni.assessor import Assessor, AssessResult from .model_factory import CurveModel @@ -37,7 +38,7 @@ class CurvefittingAssessor(Assessor): threshold: float The threshold that we decide to early stop the worse performance curve. """ - def __init__(self, epoch_num=20, optimize_mode='maximize', start_step=6, threshold=0.95): + def __init__(self, epoch_num=20, optimize_mode='maximize', start_step=6, threshold=0.95, gap=1): if start_step <= 0: logger.warning('It\'s recommended to set start_step to a positive number') # Record the target position we predict @@ -54,6 +55,10 @@ def __init__(self, epoch_num=20, optimize_mode='maximize', start_step=6, thresho self.start_step = start_step # Record the compared threshold self.threshold = threshold + # Record the number of gap + self.gap = gap + # Record the number of times of judgments + self.judgment_num = 0 # Record the best performance self.set_best_performance = False self.completed_best_performance = None @@ -100,14 +105,20 @@ def assess_trial(self, trial_job_id, trial_history): Exception unrecognize exception in curvefitting_assessor """ + self.trial_job_id = trial_job_id self.trial_history = trial_history + if not self.set_best_performance: + return AssessResult.Good curr_step = len(trial_history) if curr_step < self.start_step: return AssessResult.Good - if not self.set_best_performance: + if (curr_step - self.start_step) // self.gap <= self.judgment_num: return AssessResult.Good + self.judgment_num = (curr_step - self.start_step) // self.gap try: + start_time = datetime.datetime.now() + # Predict the final result curvemodel = CurveModel(self.target_pos) predict_y = curvemodel.predict(trial_history) logger.info('Prediction done. Trial job id = ', trial_job_id, '. Predict value = ', predict_y) @@ -115,6 +126,11 @@ def assess_trial(self, trial_job_id, trial_history): logger.info('wait for more information to predict precisely') return AssessResult.Good standard_performance = self.completed_best_performance * self.threshold + + end_time = datetime.datetime.now() + if (end_time - start_time).seconds > 60: + logger.warning('Curve Fitting Assessor Runtime Exceeds 60s, Trial Id = ', self.trial_job_id, 'Trial History = ', self.trial_history) + if self.higher_better: if predict_y > standard_performance: return AssessResult.Good diff --git a/src/sdk/pynni/nni/curvefitting_assessor/model_factory.py b/src/sdk/pynni/nni/curvefitting_assessor/model_factory.py index 1d6d3794c6..15f47e54b4 100644 --- a/src/sdk/pynni/nni/curvefitting_assessor/model_factory.py +++ b/src/sdk/pynni/nni/curvefitting_assessor/model_factory.py @@ -22,8 +22,6 @@ # Number of curve functions we prepared, more details can be found in "curvefunctions.py" NUM_OF_FUNCTIONS = 12 -# Maximum number of iterations when fitting the curve optimal parameters -MAXFEV = 1000000 # Number of simulation time when we do MCMC sampling NUM_OF_SIMULATION_TIME = 20 # Number of samples we select when we do MCMC sampling @@ -65,17 +63,18 @@ def fit_theta(self): for i in range(NUM_OF_FUNCTIONS): model = curve_combination_models[i] try: + # The maximum number of iterations to fit is 100*(N+1), where N is the number of elements in `x0`. if model_para_num[model] == 2: - a, b = optimize.curve_fit(all_models[model], x, y, maxfev=MAXFEV)[0] + a, b = optimize.curve_fit(all_models[model], x, y)[0] model_para[model][0] = a model_para[model][1] = b elif model_para_num[model] == 3: - a, b, c = optimize.curve_fit(all_models[model], x, y, maxfev=MAXFEV)[0] + a, b, c = optimize.curve_fit(all_models[model], x, y)[0] model_para[model][0] = a model_para[model][1] = b model_para[model][2] = c elif model_para_num[model] == 4: - a, b, c, d = optimize.curve_fit(all_models[model], x, y, maxfev=MAXFEV)[0] + a, b, c, d = optimize.curve_fit(all_models[model], x, y)[0] model_para[model][0] = a model_para[model][1] = b model_para[model][2] = c @@ -110,7 +109,8 @@ def filter_curve(self): std = np.std(predict_data) for model in tmp_model: y = self.predict_y(model, self.target_pos) - if y < median + 3 * std and y > median - 3 * std: + epsilon = self.point_num / 10 * std + if y < median + epsilon and y > median - epsilon: self.effective_model.append(model) self.effective_model_num = len(self.effective_model) logger.info('List of effective model: ', self.effective_model) diff --git a/tools/nni_cmd/config_schema.py b/tools/nni_cmd/config_schema.py index 29bcf1aa2e..e982a0fb38 100644 --- a/tools/nni_cmd/config_schema.py +++ b/tools/nni_cmd/config_schema.py @@ -100,7 +100,8 @@ 'epoch_num': And(int, lambda x: 0 <= x <= 9999), Optional('optimize_mode'): Or('maximize', 'minimize'), Optional('start_step'): And(int, lambda x: 0 <= x <= 9999), - Optional('threshold'): And(float, lambda x: 0.0 <= x <= 9999.0) + Optional('threshold'): And(float, lambda x: 0.0 <= x <= 9999.0), + Optional('gap'): And(int, lambda x: 1 <= x <= 9999) }, Optional('gpuNum'): And(int, lambda x: 0 <= x <= 99999) },{