Merge pull request #132 from Microsoft/master

Improve the performance of the curve fitting assessor (#717)

docs/en_US/Builtin_Assessors.md

@@ -63,6 +63,7 @@ It is applicable in a wide range of performance curves, thus, can be used in var
 * **optimize_mode** (*maximize or minimize, optional, default = maximize*) - If 'maximize', assessor will **stop** the trial with smaller expectation. If 'minimize', assessor will **stop** the trial with larger expectation.
 * **start_step** (*int, optional, default = 6*) - A trial is determined to be stopped or not, we start to predict only after receiving start_step number of reported intermediate results.
 * **threshold** (*float, optional, default = 0.95*) - The threshold that we decide to early stop the worse performance curve. For example: if threshold = 0.95, optimize_mode = maximize, best performance in the history is 0.9, then we will stop the trial which predict value is lower than 0.95 * 0.9 = 0.855.
+* **gap** (*int, optional, default = 1*) - The gap interval between Assesor judgements. For example: if gap = 2, start_step = 6, then we will assess the result when we get 6, 8, 10, 12...intermedian result.
 
 **Usage example:**
 
@@ -75,4 +76,5 @@ assessor:
       optimize_mode: maximize
       start_step: 6
       threshold: 0.95
+      gap: 1
 ```

docs/en_US/curvefittingAssessor.md

@@ -57,6 +57,10 @@ assessor:
       * The default value of threshold is 0.95.
       # Kindly reminds that if you choose minimize mode, please adjust the value of threshold >= 1.0 (e.g threshold=1.1)
       threshold: 0.95
+      # (optional) The gap interval between Assesor judgements.
+      # For example: if gap = 2, start_step = 6, then we will assess the result when we get 6, 8, 10, 12...intermedian result.
+      * The default value of gap is 1.
+      gap: 1
 ```
 
 ## 3. File Structure

src/sdk/pynni/nni/curvefitting_assessor/curvefitting_assessor.py

@@ -16,6 +16,7 @@
 # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 
 import logging
+import datetime
 from nni.assessor import Assessor, AssessResult
 from .model_factory import CurveModel
 
@@ -37,7 +38,7 @@ class CurvefittingAssessor(Assessor):
     threshold: float
         The threshold that we decide to early stop the worse performance curve.
     """
-    def __init__(self, epoch_num=20, optimize_mode='maximize', start_step=6, threshold=0.95):
+    def __init__(self, epoch_num=20, optimize_mode='maximize', start_step=6, threshold=0.95, gap=1):
         if start_step <= 0:
             logger.warning('It\'s recommended to set start_step to a positive number')
         # Record the target position we predict
@@ -54,6 +55,10 @@ def __init__(self, epoch_num=20, optimize_mode='maximize', start_step=6, thresho
         self.start_step = start_step
         # Record the compared threshold
         self.threshold = threshold
+        # Record the number of gap
+        self.gap = gap
+        # Record the number of times of judgments
+        self.judgment_num = 0
         # Record the best performance
         self.set_best_performance = False
         self.completed_best_performance = None
@@ -100,21 +105,32 @@ def assess_trial(self, trial_job_id, trial_history):
         Exception
             unrecognize exception in curvefitting_assessor
         """
+        self.trial_job_id = trial_job_id
         self.trial_history = trial_history
+        if not self.set_best_performance:
+            return AssessResult.Good
         curr_step = len(trial_history)
         if curr_step < self.start_step:
             return AssessResult.Good
-        if not self.set_best_performance:
+        if (curr_step - self.start_step) // self.gap <= self.judgment_num:
             return AssessResult.Good
+        self.judgment_num = (curr_step - self.start_step) // self.gap
 
         try:
+            start_time = datetime.datetime.now()
+            # Predict the final result
             curvemodel = CurveModel(self.target_pos)
             predict_y = curvemodel.predict(trial_history)
             logger.info('Prediction done. Trial job id = ', trial_job_id, '. Predict value = ', predict_y)
             if predict_y is None:
                 logger.info('wait for more information to predict precisely')
                 return AssessResult.Good
             standard_performance = self.completed_best_performance * self.threshold
+
+            end_time = datetime.datetime.now()
+            if (end_time - start_time).seconds > 60:
+                logger.warning('Curve Fitting Assessor Runtime Exceeds 60s, Trial Id = ', self.trial_job_id, 'Trial History = ', self.trial_history)
+
             if self.higher_better:
                 if predict_y > standard_performance:
                     return AssessResult.Good

src/sdk/pynni/nni/curvefitting_assessor/model_factory.py

@@ -22,8 +22,6 @@
 
 # Number of curve functions we prepared, more details can be found in "curvefunctions.py"
 NUM_OF_FUNCTIONS = 12
-# Maximum number of iterations when fitting the curve optimal parameters
-MAXFEV = 1000000
 # Number of simulation time when we do MCMC sampling
 NUM_OF_SIMULATION_TIME = 20
 # Number of samples we select when we do MCMC sampling
@@ -65,17 +63,18 @@ def fit_theta(self):
         for i in range(NUM_OF_FUNCTIONS):
             model = curve_combination_models[i]
             try:
+                # The maximum number of iterations to fit is 100*(N+1), where N is the number of elements in `x0`.
                 if model_para_num[model] == 2:
-                    a, b = optimize.curve_fit(all_models[model], x, y, maxfev=MAXFEV)[0]
+                    a, b = optimize.curve_fit(all_models[model], x, y)[0]
                     model_para[model][0] = a
                     model_para[model][1] = b
                 elif model_para_num[model] == 3:
-                    a, b, c = optimize.curve_fit(all_models[model], x, y, maxfev=MAXFEV)[0]
+                    a, b, c = optimize.curve_fit(all_models[model], x, y)[0]
                     model_para[model][0] = a
                     model_para[model][1] = b
                     model_para[model][2] = c
                 elif model_para_num[model] == 4:
-                    a, b, c, d = optimize.curve_fit(all_models[model], x, y, maxfev=MAXFEV)[0]
+                    a, b, c, d = optimize.curve_fit(all_models[model], x, y)[0]
                     model_para[model][0] = a
                     model_para[model][1] = b
                     model_para[model][2] = c
@@ -110,7 +109,8 @@ def filter_curve(self):
         std = np.std(predict_data)
         for model in tmp_model:
             y = self.predict_y(model, self.target_pos)
-            if y < median + 3 * std and y > median - 3 * std:
+            epsilon = self.point_num / 10 * std
+            if y < median + epsilon and y > median - epsilon:
                 self.effective_model.append(model)
         self.effective_model_num = len(self.effective_model)
         logger.info('List of effective model: ', self.effective_model)

tools/nni_cmd/config_schema.py

@@ -100,7 +100,8 @@
         'epoch_num': And(int, lambda x: 0 <= x <= 9999),
         Optional('optimize_mode'): Or('maximize', 'minimize'),
         Optional('start_step'): And(int, lambda x: 0 <= x <= 9999),
-        Optional('threshold'): And(float, lambda x: 0.0 <= x <= 9999.0)
+        Optional('threshold'): And(float, lambda x: 0.0 <= x <= 9999.0),
+        Optional('gap'): And(int, lambda x: 1 <= x <= 9999)
     },
     Optional('gpuNum'): And(int, lambda x: 0 <= x <= 99999)
 },{