diff --git a/src/sdk/pynni/nni/msg_dispatcher.py b/src/sdk/pynni/nni/msg_dispatcher.py index 0beda3f154..3abf8066c6 100644 --- a/src/sdk/pynni/nni/msg_dispatcher.py +++ b/src/sdk/pynni/nni/msg_dispatcher.py @@ -171,15 +171,15 @@ def _handle_final_metric_data(self, data): id_ = data['parameter_id'] value = data['value'] if id_ in _customized_parameter_ids: - if multi_phase_enabled(): - self.tuner.receive_customized_trial_result(id_, _trial_params[id_], value, trial_job_id=data['trial_job_id']) - else: - self.tuner.receive_customized_trial_result(id_, _trial_params[id_], value) + if not hasattr(self.tuner, '_accept_customized'): + self.tuner._accept_customized = False + if not self.tuner._accept_customized: + _logger.info('Customized trial job %s ignored by tuner', id_) + return + customized = True else: - if multi_phase_enabled(): - self.tuner.receive_trial_result(id_, _trial_params[id_], value, trial_job_id=data['trial_job_id']) - else: - self.tuner.receive_trial_result(id_, _trial_params[id_], value) + customized = False + self.tuner.receive_trial_result(id_, _trial_params[id_], value, customized=customized, trial_job_id=data.get('trial_job_id')) def _handle_intermediate_metric_data(self, data): """Call assessor to process intermediate results diff --git a/src/sdk/pynni/nni/tuner.py b/src/sdk/pynni/nni/tuner.py index 0d995f94c5..e8c345b3f3 100644 --- a/src/sdk/pynni/nni/tuner.py +++ b/src/sdk/pynni/nni/tuner.py @@ -57,19 +57,22 @@ def generate_multiple_parameters(self, parameter_id_list, **kwargs): def receive_trial_result(self, parameter_id, parameters, value, **kwargs): """Invoked when a trial reports its final result. Must override. + By default this only reports results of algorithm-generated hyper-parameters. + Use `accept_customized_trials()` to receive results from user-added parameters. parameter_id: int parameters: object created by 'generate_parameters()' - reward: object reported by trial + value: object reported by trial + customized: bool, true if the trial is created from web UI, false if generated by algorithm + trial_job_id: str, only available in multiphase mode. """ raise NotImplementedError('Tuner: receive_trial_result not implemented') - def receive_customized_trial_result(self, parameter_id, parameters, value, **kwargs): - """Invoked when a trial added by WebUI reports its final result. Do nothing by default. - parameter_id: int - parameters: object created by user - value: object reported by trial + def accept_customized_trials(self, accept=True): + """Enable or disable receiving results of user-added hyper-parameters. + By default `receive_trial_result()` will only receive results of algorithm-generated hyper-parameters. + If tuners want to receive those of customized parameters as well, they can call this function in `__init__()`. """ - _logger.info('Customized trial job %s ignored by tuner', parameter_id) + self._accept_customized = accept def trial_end(self, parameter_id, success, **kwargs): """Invoked when a trial is completed or terminated. Do nothing by default. diff --git a/src/sdk/pynni/tests/test_tuner.py b/src/sdk/pynni/tests/test_tuner.py index 41e80cfa6d..c1fd3594ee 100644 --- a/src/sdk/pynni/tests/test_tuner.py +++ b/src/sdk/pynni/tests/test_tuner.py @@ -34,6 +34,7 @@ def __init__(self): self.param = 0 self.trial_results = [ ] self.search_space = None + self.accept_customized_trials() def generate_parameters(self, parameter_id, **kwargs): # report Tuner's internal states to generated parameters, @@ -45,13 +46,9 @@ def generate_parameters(self, parameter_id, **kwargs): 'search_space': self.search_space } - def receive_trial_result(self, parameter_id, parameters, value, **kwargs): + def receive_trial_result(self, parameter_id, parameters, value, customized, **kwargs): reward = extract_scalar_reward(value) - self.trial_results.append((parameter_id, parameters['param'], reward, False)) - - def receive_customized_trial_result(self, parameter_id, parameters, value): - reward = extract_scalar_reward(value) - self.trial_results.append((parameter_id, parameters['param'], reward, True)) + self.trial_results.append((parameter_id, parameters['param'], reward, customized)) def update_search_space(self, search_space): self.search_space = search_space