diff --git a/autoPyTorch/api/base_task.py b/autoPyTorch/api/base_task.py index 3973649e7..ab71a761a 100644 --- a/autoPyTorch/api/base_task.py +++ b/autoPyTorch/api/base_task.py @@ -29,6 +29,7 @@ from smac.stats.stats import Stats from smac.tae import StatusType +from autoPyTorch.api.results_manager import ResultsManager, SearchResults from autoPyTorch.automl_common.common.utils.backend import Backend, create from autoPyTorch.constants import ( REGRESSION_TASKS, @@ -192,12 +193,13 @@ def __init__( self.search_space: Optional[ConfigurationSpace] = None self._dataset_requirements: Optional[List[FitRequirement]] = None self._metric: Optional[autoPyTorchMetric] = None + self._scoring_functions: Optional[List[autoPyTorchMetric]] = None self._logger: Optional[PicklableClientLogger] = None - self.run_history: RunHistory = RunHistory() - self.trajectory: Optional[List] = None self.dataset_name: Optional[str] = None self.cv_models_: Dict = {} + self._results_manager = ResultsManager() + # By default try to use the TCP logging port or get a new port self._logger_port = logging.handlers.DEFAULT_TCP_LOGGING_PORT @@ -240,6 +242,18 @@ def build_pipeline(self, dataset_properties: Dict[str, Any]) -> BasePipeline: """ raise NotImplementedError + @property + def run_history(self) -> RunHistory: + return self._results_manager.run_history + + @property + def ensemble_performance_history(self) -> List[Dict[str, Any]]: + return self._results_manager.ensemble_performance_history + + @property + def trajectory(self) -> Optional[List]: + return self._results_manager.trajectory + def set_pipeline_config(self, **pipeline_config_kwargs: Any) -> None: """ Check whether arguments are valid and @@ -883,6 +897,12 @@ def _search( self.pipeline_options['optimize_metric'] = optimize_metric + if all_supported_metrics: + self._scoring_functions = get_metrics(dataset_properties=dataset_properties, + all_supported_metrics=True) + else: + self._scoring_functions = [self._metric] + self.search_space = self.get_search_space(dataset) # Incorporate budget to pipeline config @@ -1037,12 +1057,14 @@ def _search( pynisher_context=self._multiprocessing_context, ) try: - run_history, self.trajectory, budget_type = \ + run_history, self._results_manager.trajectory, budget_type = \ _proc_smac.run_smbo(func=tae_func) self.run_history.update(run_history, DataOrigin.INTERNAL) trajectory_filename = os.path.join( self._backend.get_smac_output_directory_for_run(self.seed), 'trajectory.json') + + assert self.trajectory is not None # mypy check saveable_trajectory = \ [list(entry[:2]) + [entry[2].get_dictionary()] + list(entry[3:]) for entry in self.trajectory] @@ -1059,7 +1081,7 @@ def _search( self._logger.info("Starting Shutdown") if proc_ensemble is not None: - self.ensemble_performance_history = list(proc_ensemble.history) + self._results_manager.ensemble_performance_history = list(proc_ensemble.history) if len(proc_ensemble.futures) > 0: # Also add ensemble runs that did not finish within smac time @@ -1068,7 +1090,7 @@ def _search( result = proc_ensemble.futures.pop().result() if result: ensemble_history, _, _, _ = result - self.ensemble_performance_history.extend(ensemble_history) + self._results_manager.ensemble_performance_history.extend(ensemble_history) self._logger.info("Ensemble script finished, continue shutdown.") # save the ensemble performance history file @@ -1356,28 +1378,12 @@ def get_incumbent_results( The incumbent configuration Dict[str, Union[int, str, float]]: Additional information about the run of the incumbent configuration. - """ - assert self.run_history is not None, "No Run History found, search has not been called." - if self.run_history.empty(): - raise ValueError("Run History is empty. Something went wrong, " - "smac was not able to fit any model?") - - run_history_data = self.run_history.data - if not include_traditional: - # traditional classifiers have trainer_configuration in their additional info - run_history_data = dict( - filter(lambda elem: elem[1].status == StatusType.SUCCESS and elem[1]. - additional_info is not None and elem[1]. - additional_info['configuration_origin'] != 'traditional', - run_history_data.items())) - run_history_data = dict( - filter(lambda elem: 'SUCCESS' in str(elem[1].status), run_history_data.items())) - sorted_runvalue_by_cost = sorted(run_history_data.items(), key=lambda item: item[1].cost) - incumbent_run_key, incumbent_run_value = sorted_runvalue_by_cost[0] - incumbent_config = self.run_history.ids_config[incumbent_run_key.config_id] - incumbent_results = incumbent_run_value.additional_info - return incumbent_config, incumbent_results + + if self._metric is None: + raise RuntimeError("`search_results` is only available after a search has finished.") + + return self._results_manager.get_incumbent_results(metric=self._metric, include_traditional=include_traditional) def get_models_with_weights(self) -> List: if self.models_ is None or len(self.models_) == 0 or \ @@ -1417,3 +1423,43 @@ def _print_debug_info_to_log(self) -> None: self._logger.debug(' multiprocessing_context: %s', str(self._multiprocessing_context)) for key, value in vars(self).items(): self._logger.debug(f"\t{key}->{value}") + + def get_search_results(self) -> SearchResults: + """ + Get the interface to obtain the search results easily. + """ + if self._scoring_functions is None or self._metric is None: + raise RuntimeError("`search_results` is only available after a search has finished.") + + return self._results_manager.get_search_results( + metric=self._metric, + scoring_functions=self._scoring_functions + ) + + def sprint_statistics(self) -> str: + """ + Prints statistics about the SMAC search. + + These statistics include: + + 1. Optimisation Metric + 2. Best Optimisation score achieved by individual pipelines + 3. Total number of target algorithm runs + 4. Total number of successful target algorithm runs + 5. Total number of crashed target algorithm runs + 6. Total number of target algorithm runs that exceeded the time limit + 7. Total number of successful target algorithm runs that exceeded the memory limit + + Returns: + (str): + Formatted string with statistics + """ + if self._scoring_functions is None or self._metric is None: + raise RuntimeError("`search_results` is only available after a search has finished.") + + assert self.dataset_name is not None # my check + return self._results_manager.sprint_statistics( + dataset_name=self.dataset_name, + scoring_functions=self._scoring_functions, + metric=self._metric + ) diff --git a/autoPyTorch/api/results_manager.py b/autoPyTorch/api/results_manager.py new file mode 100644 index 000000000..e52d21613 --- /dev/null +++ b/autoPyTorch/api/results_manager.py @@ -0,0 +1,326 @@ +import io +from typing import Any, Dict, List, Optional, Tuple, Union + +from ConfigSpace.configuration_space import Configuration + +import numpy as np + +import scipy + +from smac.runhistory.runhistory import RunHistory, RunValue +from smac.tae import StatusType +from smac.utils.io.traj_logging import TrajEntry + +from autoPyTorch.pipeline.components.training.metrics.base import autoPyTorchMetric + + +# TODO remove StatusType.RUNNING at some point in the future when the new SMAC 0.13.2 +# is the new minimum required version! +STATUS2MSG = { + StatusType.SUCCESS: 'Success', + StatusType.DONOTADVANCE: 'Success (but did not advance to higher budget)', + StatusType.TIMEOUT: 'Timeout', + StatusType.CRASHED: 'Crash', + StatusType.ABORT: 'Abort', + StatusType.MEMOUT: 'Memory out' +} + + +def cost2metric(cost: float, metric: autoPyTorchMetric) -> float: + """ + Revert cost metric evaluated in SMAC to the original metric. + + The conversion is defined in: + autoPyTorch/pipeline/components/training/metrics/utils.py::calculate_loss + cost = metric._optimum - metric._sign * original_metric_value + ==> original_metric_value = metric._sign * (metric._optimum - cost) + """ + return metric._sign * (metric._optimum - cost) + + +def _extract_metrics_info( + run_value: RunValue, + scoring_functions: List[autoPyTorchMetric] +) -> Dict[str, float]: + """ + Extract the metric information given a run_value + and a list of metrics of interest. + + Args: + run_value (RunValue): + The information for each config evaluation. + scoring_functions (List[autoPyTorchMetric]): + The list of metrics to retrieve the info. + """ + + if run_value.status not in (StatusType.SUCCESS, StatusType.DONOTADVANCE): + # Additional info for metrics is not available in this case. + return {metric.name: np.nan for metric in scoring_functions} + + cost_info = run_value.additional_info['opt_loss'] + avail_metrics = cost_info.keys() + + return { + metric.name: cost2metric(cost=cost_info[metric.name], metric=metric) + if metric.name in avail_metrics else np.nan + for metric in scoring_functions + } + + +class SearchResults: + def __init__( + self, + metric: autoPyTorchMetric, + scoring_functions: List[autoPyTorchMetric], + run_history: RunHistory + ): + self.metric_dict: Dict[str, List[float]] = { + metric.name: [] + for metric in scoring_functions + } + self._opt_scores: List[float] = [] + self._fit_times: List[float] = [] + self.configs: List[Configuration] = [] + self.status_types: List[str] = [] + self.budgets: List[float] = [] + self.config_ids: List[int] = [] + self.is_traditionals: List[bool] = [] + self.additional_infos: List[Optional[Dict[str, Any]]] = [] + self.rank_test_scores: np.ndarray = np.array([]) + self._scoring_functions = scoring_functions + self._metric = metric + + self._extract_results_from_run_history(run_history) + + @property + def opt_scores(self) -> np.ndarray: + return np.asarray(self._opt_scores) + + @property + def fit_times(self) -> np.ndarray: + return np.asarray(self._fit_times) + + def update( + self, + config: Configuration, + status: str, + budget: float, + fit_time: float, + config_id: int, + is_traditional: bool, + additional_info: Dict[str, Any], + score: float, + metric_info: Dict[str, float] + ) -> None: + + self.status_types.append(status) + self.configs.append(config) + self.budgets.append(budget) + self.config_ids.append(config_id) + self.is_traditionals.append(is_traditional) + self.additional_infos.append(additional_info) + self._fit_times.append(fit_time) + self._opt_scores.append(score) + + for metric_name, val in metric_info.items(): + self.metric_dict[metric_name].append(val) + + def clear(self) -> None: + self._opt_scores = [] + self._fit_times = [] + self.configs = [] + self.status_types = [] + self.budgets = [] + self.config_ids = [] + self.additional_infos = [] + self.is_traditionals = [] + self.rank_test_scores = np.array([]) + + def _extract_results_from_run_history(self, run_history: RunHistory) -> None: + """ + Extract the information to match this class format. + + Args: + run_history (RunHistory): + The history of config evals from SMAC. + """ + + self.clear() # Delete cache before the extraction + + for run_key, run_value in run_history.data.items(): + config_id = run_key.config_id + config = run_history.ids_config[config_id] + + status_msg = STATUS2MSG.get(run_value.status, None) + if run_value.status in (StatusType.STOP, StatusType.RUNNING): + continue + elif status_msg is None: + raise ValueError(f'Unexpected run status: {run_value.status}') + + is_traditional = False # If run is not successful, unsure ==> not True ==> False + if run_value.additional_info is not None: + is_traditional = run_value.additional_info['configuration_origin'] == 'traditional' + + self.update( + status=status_msg, + config=config, + budget=run_key.budget, + fit_time=run_value.time, + score=cost2metric(cost=run_value.cost, metric=self._metric), + metric_info=_extract_metrics_info(run_value=run_value, scoring_functions=self._scoring_functions), + is_traditional=is_traditional, + additional_info=run_value.additional_info, + config_id=config_id + ) + + self.rank_test_scores = scipy.stats.rankdata( + -1 * self._metric._sign * self.opt_scores, # rank order + method='min' + ) + + +class ResultsManager: + def __init__(self, *args: Any, **kwargs: Any): + """ + Attributes: + run_history (RunHistory): + A `SMAC Runshistory `_ + object that holds information about the runs of the target algorithm made during search + ensemble_performance_history (List[Dict[str, Any]]): + The list of ensemble performance in the optimization. + The list includes the `timestamp`, `result on train set`, and `result on test set` + trajectory (List[TrajEntry]): + A list of all incumbent configurations during search + """ + self.run_history: RunHistory = RunHistory() + self.ensemble_performance_history: List[Dict[str, Any]] = [] + self.trajectory: List[TrajEntry] = [] + + def _check_run_history(self) -> None: + if self.run_history is None: + raise RuntimeError("No Run History found, search has not been called.") + + if self.run_history.empty(): + raise RuntimeError("Run History is empty. Something went wrong, " + "SMAC was not able to fit any model?") + + def get_incumbent_results( + self, + metric: autoPyTorchMetric, + include_traditional: bool = False + ) -> Tuple[Configuration, Dict[str, Union[int, str, float]]]: + """ + Get Incumbent config and the corresponding results + + Args: + metric (autoPyTorchMetric): + A metric that is evaluated when searching with fit AutoPytorch. + include_traditional (bool): + Whether to include results from tradtional pipelines + + Returns: + Configuration (CS.ConfigurationSpace): + The incumbent configuration + Dict[str, Union[int, str, float]]: + Additional information about the run of the incumbent configuration. + """ + self._check_run_history() + + results = SearchResults(metric=metric, scoring_functions=[], run_history=self.run_history) + + if not include_traditional: + non_traditional = ~np.array(results.is_traditionals) + scores = results.opt_scores[non_traditional] + indices = np.arange(len(results.configs))[non_traditional] + else: + scores = results.opt_scores + indices = np.arange(len(results.configs)) + + incumbent_idx = indices[np.nanargmax(metric._sign * scores)] + incumbent_config = results.configs[incumbent_idx] + incumbent_results = results.additional_infos[incumbent_idx] + + assert incumbent_results is not None # mypy check + return incumbent_config, incumbent_results + + def get_search_results( + self, + scoring_functions: List[autoPyTorchMetric], + metric: autoPyTorchMetric + ) -> SearchResults: + """ + This attribute is populated with data from `self.run_history` + and contains information about the configurations, and their + corresponding metric results, status of run, parameters and + the budget + + Args: + scoring_functions (List[autoPyTorchMetric]): + Metrics to show in the results. + metric (autoPyTorchMetric): + A metric that is evaluated when searching with fit AutoPytorch. + + Returns: + SearchResults: + An instance that contains the results from search + """ + self._check_run_history() + return SearchResults(metric=metric, scoring_functions=scoring_functions, run_history=self.run_history) + + def sprint_statistics( + self, + dataset_name: str, + scoring_functions: List[autoPyTorchMetric], + metric: autoPyTorchMetric + ) -> str: + """ + Prints statistics about the SMAC search. + + These statistics include: + + 1. Optimisation Metric + 2. Best Optimisation score achieved by individual pipelines + 3. Total number of target algorithm runs + 4. Total number of successful target algorithm runs + 5. Total number of crashed target algorithm runs + 6. Total number of target algorithm runs that exceeded the time limit + 7. Total number of successful target algorithm runs that exceeded the memory limit + + Args: + dataset_name (str): + The dataset name that was used in the run. + scoring_functions (List[autoPyTorchMetric]): + Metrics to show in the results. + metric (autoPyTorchMetric): + A metric that is evaluated when searching with fit AutoPytorch. + + Returns: + (str): + Formatted string with statistics + """ + search_results = self.get_search_results(scoring_functions, metric) + success_msgs = (STATUS2MSG[StatusType.SUCCESS], STATUS2MSG[StatusType.DONOTADVANCE]) + sio = io.StringIO() + sio.write("autoPyTorch results:\n") + sio.write(f"\tDataset name: {dataset_name}\n") + sio.write(f"\tOptimisation Metric: {metric}\n") + + num_runs = len(search_results.status_types) + num_success = sum([s in success_msgs for s in search_results.status_types]) + num_crash = sum([s == STATUS2MSG[StatusType.CRASHED] for s in search_results.status_types]) + num_timeout = sum([s == STATUS2MSG[StatusType.TIMEOUT] for s in search_results.status_types]) + num_memout = sum([s == STATUS2MSG[StatusType.MEMOUT] for s in search_results.status_types]) + + if num_success > 0: + best_score = metric._sign * np.nanmax(metric._sign * search_results.opt_scores) + sio.write(f"\tBest validation score: {best_score}\n") + + sio.write(f"\tNumber of target algorithm runs: {num_runs}\n") + sio.write(f"\tNumber of successful target algorithm runs: {num_success}\n") + sio.write(f"\tNumber of crashed target algorithm runs: {num_crash}\n") + sio.write(f"\tNumber of target algorithms that exceeded the time " + f"limit: {num_timeout}\n") + sio.write(f"\tNumber of target algorithms that exceeded the memory " + f"limit: {num_memout}\n") + + return sio.getvalue() diff --git a/autoPyTorch/api/tabular_classification.py b/autoPyTorch/api/tabular_classification.py index 7eb25009d..3be138525 100644 --- a/autoPyTorch/api/tabular_classification.py +++ b/autoPyTorch/api/tabular_classification.py @@ -274,6 +274,7 @@ def search( X=X_train, Y=y_train, X_test=X_test, Y_test=y_test, validator=self.InputValidator, + dataset_name=dataset_name, resampling_strategy=self.resampling_strategy, resampling_strategy_args=self.resampling_strategy_args, ) diff --git a/autoPyTorch/api/tabular_regression.py b/autoPyTorch/api/tabular_regression.py index 74e537c99..3ad055028 100644 --- a/autoPyTorch/api/tabular_regression.py +++ b/autoPyTorch/api/tabular_regression.py @@ -275,6 +275,7 @@ def search( X=X_train, Y=y_train, X_test=X_test, Y_test=y_test, validator=self.InputValidator, + dataset_name=dataset_name, resampling_strategy=self.resampling_strategy, resampling_strategy_args=self.resampling_strategy_args, ) diff --git a/autoPyTorch/evaluation/abstract_evaluator.py b/autoPyTorch/evaluation/abstract_evaluator.py index b926a50a7..027c7211a 100644 --- a/autoPyTorch/evaluation/abstract_evaluator.py +++ b/autoPyTorch/evaluation/abstract_evaluator.py @@ -735,7 +735,7 @@ def calculate_auxiliary_losses( self, Y_valid_pred: np.ndarray, Y_test_pred: np.ndarray, - ) -> Tuple[Optional[float], Optional[float]]: + ) -> Tuple[Optional[Dict[str, float]], Optional[Dict[str, float]]]: """ A helper function to calculate the performance estimate of the current pipeline in the user provided validation/test set. @@ -747,29 +747,26 @@ def calculate_auxiliary_losses( Y_test_pred (np.ndarray): predictions on a test set provided by the user, matching self.y_test + Returns: - validation_loss (Optional[float]): - The validation loss under the optimization metric - stored in self.metric - test_loss (Optional[float]]): - The test loss under the optimization metric - stored in self.metric + validation_loss_dict (Optional[Dict[str, float]]): + Various validation losses available. + test_loss_dict (Optional[Dict[str, float]]): + Various test losses available. """ - validation_loss: Optional[float] = None + validation_loss_dict: Optional[Dict[str, float]] = None if Y_valid_pred is not None: if self.y_valid is not None: validation_loss_dict = self._loss(self.y_valid, Y_valid_pred) - validation_loss = validation_loss_dict[self.metric.name] - test_loss: Optional[float] = None + test_loss_dict: Optional[Dict[str, float]] = None if Y_test_pred is not None: if self.y_test is not None: test_loss_dict = self._loss(self.y_test, Y_test_pred) - test_loss = test_loss_dict[self.metric.name] - return validation_loss, test_loss + return validation_loss_dict, test_loss_dict def file_output( self, diff --git a/examples/20_basics/example_tabular_classification.py b/examples/20_basics/example_tabular_classification.py index 7b1aa9995..636281eff 100644 --- a/examples/20_basics/example_tabular_classification.py +++ b/examples/20_basics/example_tabular_classification.py @@ -55,6 +55,7 @@ y_train=y_train, X_test=X_test.copy(), y_test=y_test.copy(), + dataset_name='Australian', optimize_metric='accuracy', total_walltime_limit=300, func_eval_time_limit_secs=50 @@ -63,9 +64,11 @@ ############################################################################ # Print the final ensemble performance # ==================================== -print(api.run_history, api.trajectory) y_pred = api.predict(X_test) score = api.score(y_pred, y_test) print(score) # Print the final ensemble built by AutoPyTorch print(api.show_models()) + +# Print statistics from search +print(api.sprint_statistics()) diff --git a/examples/20_basics/example_tabular_regression.py b/examples/20_basics/example_tabular_regression.py index 9b4e876e9..127f26829 100644 --- a/examples/20_basics/example_tabular_regression.py +++ b/examples/20_basics/example_tabular_regression.py @@ -55,7 +55,6 @@ ############################################################################ # Print the final ensemble performance # ==================================== -print(api.run_history, api.trajectory) y_pred = api.predict(X_test) # Rescale the Neural Network predictions into the original target range @@ -64,3 +63,6 @@ print(score) # Print the final ensemble built by AutoPyTorch print(api.show_models()) + +# Print statistics from search +print(api.sprint_statistics()) diff --git a/examples/40_advanced/example_custom_configuration_space.py b/examples/40_advanced/example_custom_configuration_space.py index f552045c1..c64a4fca1 100644 --- a/examples/40_advanced/example_custom_configuration_space.py +++ b/examples/40_advanced/example_custom_configuration_space.py @@ -94,12 +94,14 @@ def get_search_space_updates(): ############################################################################ # Print the final ensemble performance # ==================================== - print(api.run_history, api.trajectory) y_pred = api.predict(X_test) score = api.score(y_pred, y_test) print(score) print(api.show_models()) + # Print statistics from search + print(api.sprint_statistics()) + ############################################################################ # Build and fit a classifier with exclude components # ================================================== @@ -125,8 +127,10 @@ def get_search_space_updates(): ############################################################################ # Print the final ensemble performance # ==================================== - print(api.run_history, api.trajectory) y_pred = api.predict(X_test) score = api.score(y_pred, y_test) print(score) print(api.show_models()) + + # Print statistics from search + print(api.sprint_statistics()) diff --git a/examples/40_advanced/example_resampling_strategy.py b/examples/40_advanced/example_resampling_strategy.py index 9fb77b76d..6735fffee 100644 --- a/examples/40_advanced/example_resampling_strategy.py +++ b/examples/40_advanced/example_resampling_strategy.py @@ -65,13 +65,15 @@ ############################################################################ # Print the final ensemble performance # ==================================== -print(api.run_history, api.trajectory) y_pred = api.predict(X_test) score = api.score(y_pred, y_test) print(score) # Print the final ensemble built by AutoPyTorch print(api.show_models()) +# Print statistics from search +print(api.sprint_statistics()) + ############################################################################ ############################################################################ @@ -98,13 +100,15 @@ ############################################################################ # Print the final ensemble performance # ==================================== -print(api.run_history, api.trajectory) y_pred = api.predict(X_test) score = api.score(y_pred, y_test) print(score) # Print the final ensemble built by AutoPyTorch print(api.show_models()) +# Print statistics from search +print(api.sprint_statistics()) + ############################################################################ ############################################################################ @@ -134,9 +138,11 @@ ############################################################################ # Print the final ensemble performance # ==================================== -print(api.run_history, api.trajectory) y_pred = api.predict(X_test) score = api.score(y_pred, y_test) print(score) # Print the final ensemble built by AutoPyTorch print(api.show_models()) + +# Print statistics from search +print(api.sprint_statistics()) diff --git a/examples/40_advanced/example_run_with_portfolio.py b/examples/40_advanced/example_run_with_portfolio.py index 4109a2378..01d8bef15 100644 --- a/examples/40_advanced/example_run_with_portfolio.py +++ b/examples/40_advanced/example_run_with_portfolio.py @@ -63,9 +63,11 @@ ############################################################################ # Print the final ensemble performance # ==================================== - print(api.run_history, api.trajectory) y_pred = api.predict(X_test) score = api.score(y_pred, y_test) print(score) # Print the final ensemble built by AutoPyTorch print(api.show_models()) + + # Print statistics from search + print(api.sprint_statistics()) diff --git a/examples/40_advanced/example_visualization.py b/examples/40_advanced/example_visualization.py index 107d07a47..37c1c6dc3 100644 --- a/examples/40_advanced/example_visualization.py +++ b/examples/40_advanced/example_visualization.py @@ -113,7 +113,7 @@ ), 'single_best_optimization_accuracy': accuracy._optimum - run_value.cost, 'single_best_test_accuracy': np.nan if run_value.additional_info is None else - accuracy._optimum - run_value.additional_info['test_loss'], + accuracy._optimum - run_value.additional_info['test_loss']['accuracy'], }) individual_performance_frame = pd.DataFrame(individual_performances) diff --git a/test/test_api/.tmp_api/runhistory_B.json b/test/test_api/.tmp_api/runhistory_B.json new file mode 100755 index 000000000..37e499664 --- /dev/null +++ b/test/test_api/.tmp_api/runhistory_B.json @@ -0,0 +1,1815 @@ +{ + "data": [ + [ + [ + 1, + "{\"task_id\": \"Australian\"}", + 0, + 5.555555555555555 + ], + [ + 0.15204678362573099, + 3.154788017272949, + { + "__enum__": "StatusType.SUCCESS" + }, + 1637342638.6119366, + 1637342642.7887495, + { + "opt_loss": { + "accuracy": 0.15204678362573099, + "balanced_accuracy": 0.15263157894736845, + "roc_auc": 0.08981994459833786, + "average_precision": 0.1040861796433199, + "log_loss": 0.5765479137672738, + "precision": 0.17948717948717952, + "precision_macro": 0.15425971877584788, + "precision_micro": 0.15204678362573099, + "precision_weighted": 0.15145666758569976, + "recall": 0.1578947368421053, + "recall_macro": 0.15263157894736845, + "recall_micro": 0.15204678362573099, + "recall_weighted": 0.15204678362573099, + "f1": 0.16883116883116878, + "f1_macro": 0.15356452058579717, + "f1_micro": 0.15204678362573099, + "f1_weighted": 0.15186822633631147 + }, + "duration": 3.11077618598938, + "num_run": 8, + "train_loss": { + "accuracy": 0.09537572254335258, + "balanced_accuracy": 0.10239948774980623, + "roc_auc": 0.03963198867657458, + "average_precision": 0.044469547423341305, + "log_loss": 0.28008669264774966, + "precision": 0.03731343283582089, + "precision_macro": 0.08469445226696704, + "precision_micro": 0.09537572254335258, + "precision_weighted": 0.0890765118675354, + "recall": 0.17834394904458595, + "recall_macro": 0.10239948774980623, + "recall_micro": 0.09537572254335258, + "recall_weighted": 0.09537572254335258, + "f1": 0.11340206185567014, + "f1_macro": 0.09784816309741107, + "f1_micro": 0.09537572254335258, + "f1_weighted": 0.0964096522295953 + }, + "test_loss": { + "accuracy": 0.1445086705202312, + "balanced_accuracy": 0.15356265356265353, + "roc_auc": 0.07821457821457822, + "average_precision": 0.08337624010373501, + "log_loss": 0.4291606295042948, + "precision": 0.13432835820895528, + "precision_macro": 0.14263587721768523, + "precision_micro": 0.1445086705202312, + "precision_weighted": 0.14383638574495827, + "recall": 0.21621621621621623, + "recall_macro": 0.15356265356265353, + "recall_micro": 0.1445086705202312, + "recall_weighted": 0.1445086705202312, + "f1": 0.17730496453900713, + "f1_macro": 0.14962809202560112, + "f1_micro": 0.1445086705202313, + "f1_weighted": 0.14562854397453084 + }, + "configuration_origin": "Default" + } + ] + ], + [ + [ + 2, + "{\"task_id\": \"Australian\"}", + 0, + 5.555555555555555 + ], + [ + 0.4444444444444444, + 3.2763524055480957, + { + "__enum__": "StatusType.SUCCESS" + }, + 1637342642.963385, + 1637342647.2651122, + { + "opt_loss": { + "accuracy": 0.4444444444444444, + "balanced_accuracy": 0.5, + "roc_auc": 0.25526315789473697, + "average_precision": 0.35005634879129066, + "log_loss": 1.0913122792494052, + "precision": 1.0, + "precision_macro": 0.7222222222222222, + "precision_micro": 0.4444444444444444, + "precision_weighted": 0.691358024691358, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.4444444444444444, + "recall_weighted": 0.4444444444444444, + "f1": 1.0, + "f1_macro": 0.6428571428571428, + "f1_micro": 0.4444444444444444, + "f1_weighted": 0.6031746031746031 + }, + "duration": 3.2138161659240723, + "num_run": 9, + "train_loss": { + "accuracy": 0.45375722543352603, + "balanced_accuracy": 0.5, + "roc_auc": 0.2745256630606949, + "average_precision": 0.4037230365622788, + "log_loss": 1.1229484684905306, + "precision": 1.0, + "precision_macro": 0.726878612716763, + "precision_micro": 0.45375722543352603, + "precision_weighted": 0.7016188312339203, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.45375722543352603, + "recall_weighted": 0.45375722543352603, + "f1": 1.0, + "f1_macro": 0.6467289719626168, + "f1_micro": 0.45375722543352603, + "f1_weighted": 0.6140565069418183 + }, + "test_loss": { + "accuracy": 0.4277456647398844, + "balanced_accuracy": 0.5, + "roc_auc": 0.28078078078078084, + "average_precision": 0.4240469510154605, + "log_loss": 1.0714287830118328, + "precision": 1.0, + "precision_macro": 0.7138728323699421, + "precision_micro": 0.4277456647398844, + "precision_weighted": 0.6725249757760032, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.4277456647398844, + "recall_weighted": 0.4277456647398844, + "f1": 1.0, + "f1_macro": 0.6360294117647058, + "f1_micro": 0.4277456647398844, + "f1_weighted": 0.5834325059503571 + }, + "configuration_origin": "Random Search" + } + ] + ], + [ + [ + 3, + "{\"task_id\": \"Australian\"}", + 0, + 5.555555555555555 + ], + [ + 0.5555555555555556, + 22.723600149154663, + { + "__enum__": "StatusType.SUCCESS" + }, + 1637342651.4707444, + 1637342675.2555833, + { + "opt_loss": { + "accuracy": 0.5555555555555556, + "balanced_accuracy": 0.5, + "roc_auc": 0.4924515235457063, + "average_precision": 0.5493808049535605, + "log_loss": 0.7291908971747459, + "precision": 0.5555555555555556, + "precision_macro": 0.7777777777777778, + "precision_micro": 0.5555555555555556, + "precision_weighted": 0.8024691358024691, + "recall": 0.0, + "recall_macro": 0.5, + "recall_micro": 0.5555555555555556, + "recall_weighted": 0.5555555555555556, + "f1": 0.3846153846153847, + "f1_macro": 0.6923076923076923, + "f1_micro": 0.5555555555555556, + "f1_weighted": 0.7264957264957266 + }, + "duration": 22.021637201309204, + "num_run": 10, + "train_loss": { + "accuracy": 0.546242774566474, + "balanced_accuracy": 0.5, + "roc_auc": 0.514423887035352, + "average_precision": 0.5521926852639938, + "log_loss": 0.7258427792546377, + "precision": 0.546242774566474, + "precision_macro": 0.773121387283237, + "precision_micro": 0.546242774566474, + "precision_weighted": 0.7941043803668683, + "recall": 0.0, + "recall_macro": 0.5, + "recall_micro": 0.546242774566474, + "recall_weighted": 0.546242774566474, + "f1": 0.3757455268389662, + "f1_macro": 0.6878727634194831, + "f1_micro": 0.546242774566474, + "f1_weighted": 0.7167400222939817 + }, + "test_loss": { + "accuracy": 0.5722543352601156, + "balanced_accuracy": 0.5, + "roc_auc": 0.4951542451542452, + "average_precision": 0.5698669806352692, + "log_loss": 0.7351944336312355, + "precision": 0.5722543352601156, + "precision_macro": 0.7861271676300579, + "precision_micro": 0.5722543352601156, + "precision_weighted": 0.8170336462962344, + "recall": 0.0, + "recall_macro": 0.5, + "recall_micro": 0.5722543352601156, + "recall_weighted": 0.5722543352601156, + "f1": 0.4008097165991902, + "f1_macro": 0.7004048582995951, + "f1_micro": 0.5722543352601157, + "f1_weighted": 0.7436989539210408 + }, + "configuration_origin": "Random Search (sorted)" + } + ] + ], + [ + [ + 4, + "{\"task_id\": \"Australian\"}", + 0, + 5.555555555555555 + ], + [ + 0.29824561403508776, + 4.990685224533081, + { + "__enum__": "StatusType.SUCCESS" + }, + 1637342675.317421, + 1637342681.334954, + { + "opt_loss": { + "accuracy": 0.29824561403508776, + "balanced_accuracy": 0.30263157894736836, + "roc_auc": 0.26869806094182835, + "average_precision": 0.3191125709864897, + "log_loss": 0.6374789248084465, + "precision": 0.33333333333333337, + "precision_macro": 0.30208333333333337, + "precision_micro": 0.29824561403508776, + "precision_weighted": 0.29861111111111116, + "recall": 0.3421052631578947, + "recall_macro": 0.30263157894736836, + "recall_micro": 0.29824561403508776, + "recall_weighted": 0.29824561403508776, + "f1": 0.3377483443708609, + "f1_macro": 0.30238202558857186, + "f1_micro": 0.29824561403508776, + "f1_weighted": 0.29845243461276183 + }, + "duration": 4.924501419067383, + "num_run": 11, + "train_loss": { + "accuracy": 0.3728323699421965, + "balanced_accuracy": 0.3800930138509757, + "roc_auc": 0.3314460957773059, + "average_precision": 0.3638537658311296, + "log_loss": 0.6533903728503023, + "precision": 0.4014084507042254, + "precision_macro": 0.3771748135874068, + "precision_micro": 0.3728323699421965, + "precision_weighted": 0.3749335523511692, + "recall": 0.4585987261146497, + "recall_macro": 0.3800930138509757, + "recall_micro": 0.3728323699421965, + "recall_weighted": 0.3728323699421965, + "f1": 0.43143812709030094, + "f1_macro": 0.3798412009497306, + "f1_micro": 0.3728323699421965, + "f1_weighted": 0.3750692309020478 + }, + "test_loss": { + "accuracy": 0.34104046242774566, + "balanced_accuracy": 0.35087360087360087, + "roc_auc": 0.29060879060879063, + "average_precision": 0.3873731449553385, + "log_loss": 0.6454486294805659, + "precision": 0.3943661971830986, + "precision_macro": 0.34914388290527487, + "precision_micro": 0.34104046242774566, + "precision_weighted": 0.3426088663911384, + "recall": 0.41891891891891897, + "recall_macro": 0.35087360087360087, + "recall_micro": 0.34104046242774566, + "recall_weighted": 0.34104046242774566, + "f1": 0.40689655172413797, + "f1_macro": 0.35021444501629784, + "f1_micro": 0.34104046242774566, + "f1_weighted": 0.34202338913366204 + }, + "configuration_origin": "Random Search" + } + ] + ], + [ + [ + 5, + "{\"task_id\": \"Australian\"}", + 0, + 5.555555555555555 + ], + [ + 0.4444444444444444, + 10.684926509857178, + { + "__enum__": "StatusType.SUCCESS" + }, + 1637342681.548915, + 1637342693.2717755, + { + "opt_loss": { + "accuracy": 0.4444444444444444, + "balanced_accuracy": 0.5, + "roc_auc": 0.6092797783933518, + "average_precision": 0.6129755132627962, + "log_loss": 0.6905045174715811, + "precision": 1.0, + "precision_macro": 0.7222222222222222, + "precision_micro": 0.4444444444444444, + "precision_weighted": 0.691358024691358, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.4444444444444444, + "recall_weighted": 0.4444444444444444, + "f1": 1.0, + "f1_macro": 0.6428571428571428, + "f1_micro": 0.4444444444444444, + "f1_weighted": 0.6031746031746031 + }, + "duration": 10.401196956634521, + "num_run": 12, + "train_loss": { + "accuracy": 0.45375722543352603, + "balanced_accuracy": 0.5, + "roc_auc": 0.6309102551140767, + "average_precision": 0.6325768698403712, + "log_loss": 0.691941062839045, + "precision": 1.0, + "precision_macro": 0.726878612716763, + "precision_micro": 0.45375722543352603, + "precision_weighted": 0.7016188312339203, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.45375722543352603, + "recall_weighted": 0.45375722543352603, + "f1": 1.0, + "f1_macro": 0.6467289719626168, + "f1_micro": 0.45375722543352603, + "f1_weighted": 0.6140565069418183 + }, + "test_loss": { + "accuracy": 0.4450867052023122, + "balanced_accuracy": 0.5151515151515151, + "roc_auc": 0.7676767676767677, + "average_precision": 0.7025371518831428, + "log_loss": 0.6913509909817249, + "precision": 1.0, + "precision_macro": 0.7176470588235294, + "precision_micro": 0.4450867052023122, + "precision_weighted": 0.6768446106766406, + "recall": 1.0, + "recall_macro": 0.5151515151515151, + "recall_micro": 0.4450867052023122, + "recall_weighted": 0.4450867052023122, + "f1": 1.0, + "f1_macro": 0.6431226765799256, + "f1_micro": 0.4450867052023122, + "f1_weighted": 0.5915508090336722 + }, + "configuration_origin": "Random Search" + } + ] + ], + [ + [ + 6, + "{\"task_id\": \"Australian\"}", + 0, + 5.555555555555555 + ], + [ + 0.4444444444444444, + 9.947429180145264, + { + "__enum__": "StatusType.SUCCESS" + }, + 1637342693.356699, + 1637342704.341065, + { + "opt_loss": { + "accuracy": 0.4444444444444444, + "balanced_accuracy": 0.5, + "roc_auc": 0.24930747922437668, + "average_precision": 0.31612650360994055, + "log_loss": 0.6525155201292875, + "precision": 1.0, + "precision_macro": 0.7222222222222222, + "precision_micro": 0.4444444444444444, + "precision_weighted": 0.691358024691358, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.4444444444444444, + "recall_weighted": 0.4444444444444444, + "f1": 1.0, + "f1_macro": 0.6428571428571428, + "f1_micro": 0.4444444444444444, + "f1_weighted": 0.6031746031746031 + }, + "duration": 9.76927137374878, + "num_run": 13, + "train_loss": { + "accuracy": 0.45375722543352603, + "balanced_accuracy": 0.5, + "roc_auc": 0.22427796313146642, + "average_precision": 0.2451792573360162, + "log_loss": 0.64721482587343, + "precision": 1.0, + "precision_macro": 0.726878612716763, + "precision_micro": 0.45375722543352603, + "precision_weighted": 0.7016188312339203, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.45375722543352603, + "recall_weighted": 0.45375722543352603, + "f1": 1.0, + "f1_macro": 0.6467289719626168, + "f1_micro": 0.45375722543352603, + "f1_weighted": 0.6140565069418183 + }, + "test_loss": { + "accuracy": 0.4277456647398844, + "balanced_accuracy": 0.5, + "roc_auc": 0.21730821730821726, + "average_precision": 0.24726417906930265, + "log_loss": 0.6388118587477358, + "precision": 1.0, + "precision_macro": 0.7138728323699421, + "precision_micro": 0.4277456647398844, + "precision_weighted": 0.6725249757760032, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.4277456647398844, + "recall_weighted": 0.4277456647398844, + "f1": 1.0, + "f1_macro": 0.6360294117647058, + "f1_micro": 0.4277456647398844, + "f1_weighted": 0.5834325059503571 + }, + "configuration_origin": "Random Search" + } + ] + ], + [ + [ + 7, + "{\"task_id\": \"Australian\"}", + 0, + 5.555555555555555 + ], + [ + 1.0, + 11.687273979187012, + { + "__enum__": "StatusType.CRASHED" + }, + 1637342713.4931705, + 1637342726.1866672, + { + "error": "Result queue is empty", + "exit_status": "", + "subprocess_stdout": "", + "subprocess_stderr": "Process pynisher function call:\nTraceback (most recent call last):\n File \"/home/shuhei/research/Auto-PyTorch/autoPyTorch/evaluation/tae.py\", line 39, in fit_predict_try_except_decorator\n ta(queue=queue, **kwargs)\n File \"/home/shuhei/research/Auto-PyTorch/autoPyTorch/evaluation/train_evaluator.py\", line 485, in eval_function\n evaluator.fit_predict_and_loss()\n File \"/home/shuhei/research/Auto-PyTorch/autoPyTorch/evaluation/train_evaluator.py\", line 163, in fit_predict_and_loss\n y_train_pred, y_opt_pred, y_valid_pred, y_test_pred = self._fit_and_predict(pipeline, split_id,\n File \"/home/shuhei/research/Auto-PyTorch/autoPyTorch/evaluation/train_evaluator.py\", line 337, in _fit_and_predict\n fit_and_suppress_warnings(self.logger, pipeline, X, y)\n File \"/home/shuhei/research/Auto-PyTorch/autoPyTorch/evaluation/abstract_evaluator.py\", line 321, in fit_and_suppress_warnings\n pipeline.fit(X, y)\n File \"/home/shuhei/research/Auto-PyTorch/autoPyTorch/pipeline/base_pipeline.py\", line 153, in fit\n self.fit_estimator(X, y, **fit_params)\n File \"/home/shuhei/research/Auto-PyTorch/autoPyTorch/pipeline/base_pipeline.py\", line 172, in fit_estimator\n self._final_estimator.fit(X, y, **fit_params)\n File \"/home/shuhei/research/Auto-PyTorch/autoPyTorch/pipeline/components/training/trainer/__init__.py\", line 211, in fit\n self._fit(\n File \"/home/shuhei/research/Auto-PyTorch/autoPyTorch/pipeline/components/training/trainer/__init__.py\", line 290, in _fit\n train_loss, train_metrics = self.choice.train_epoch(\n File \"/home/shuhei/research/Auto-PyTorch/autoPyTorch/pipeline/components/training/trainer/base_trainer.py\", line 303, in train_epoch\n loss, outputs = self.train_step(data, targets)\n File \"/home/shuhei/research/Auto-PyTorch/autoPyTorch/pipeline/components/training/trainer/base_trainer.py\", line 357, in train_step\n outputs = self.model(data)\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 727, in _call_impl\n result = self.forward(*input, **kwargs)\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/site-packages/torch/nn/modules/container.py\", line 117, in forward\n input = module(input)\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 727, in _call_impl\n result = self.forward(*input, **kwargs)\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/site-packages/torch/nn/modules/container.py\", line 117, in forward\n input = module(input)\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 727, in _call_impl\n result = self.forward(*input, **kwargs)\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/site-packages/torch/nn/modules/container.py\", line 117, in forward\n input = module(input)\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 727, in _call_impl\n result = self.forward(*input, **kwargs)\n File \"/home/shuhei/research/Auto-PyTorch/autoPyTorch/pipeline/components/setup/network_backbone/ResNetBackbone.py\", line 274, in forward\n x2 = self.shake_shake_layers(x)\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 727, in _call_impl\n result = self.forward(*input, **kwargs)\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/site-packages/torch/nn/modules/container.py\", line 117, in forward\n input = module(input)\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/site-packages/torch/nn/modules/module.py\", line 727, in _call_impl\n result = self.forward(*input, **kwargs)\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/site-packages/torch/nn/modules/linear.py\", line 93, in forward\n return F.linear(input, self.weight, self.bias)\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/site-packages/torch/nn/functional.py\", line 1690, in linear\n ret = torch.addmm(bias, input, weight.t())\nRuntimeError: [enforce fail at CPUAllocator.cpp:65] . DefaultCPUAllocator: can't allocate memory: you tried to allocate 713632 bytes. Error code 12 (Cannot allocate memory)\n\nDuring handling of the above exception, another exception occurred:\n\nTraceback (most recent call last):\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/multiprocessing/process.py\", line 315, in _bootstrap\n self.run()\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/multiprocessing/process.py\", line 108, in run\n self._target(*self._args, **self._kwargs)\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/site-packages/pynisher/limit_function_call.py\", line 138, in subprocess_func\n return_value = ((func(*args, **kwargs), 0))\n File \"/home/shuhei/research/Auto-PyTorch/autoPyTorch/evaluation/tae.py\", line 52, in fit_predict_try_except_decorator\n queue.put({'loss': cost_for_crash,\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/multiprocessing/queues.py\", line 88, in put\n self._start_thread()\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/multiprocessing/queues.py\", line 173, in _start_thread\n self._thread.start()\n File \"/home/shuhei/anaconda3/envs/auto-pytorch/lib/python3.8/threading.py\", line 852, in start\n _start_new_thread(self._bootstrap, ())\nRuntimeError: can't start new thread\n", + "exitcode": 1, + "configuration_origin": "Random Search (sorted)" + } + ] + ], + [ + [ + 8, + "{\"task_id\": \"Australian\"}", + 0, + 5.555555555555555 + ], + [ + 0.5555555555555556, + 8.478890419006348, + { + "__enum__": "StatusType.SUCCESS" + }, + 1637342733.815657, + 1637342743.3274522, + { + "opt_loss": { + "accuracy": 0.5555555555555556, + "balanced_accuracy": 0.5, + "roc_auc": 0.44605263157894737, + "average_precision": 0.526907034743722, + "log_loss": 0.722785997111895, + "precision": 0.5555555555555556, + "precision_macro": 0.7777777777777778, + "precision_micro": 0.5555555555555556, + "precision_weighted": 0.8024691358024691, + "recall": 0.0, + "recall_macro": 0.5, + "recall_micro": 0.5555555555555556, + "recall_weighted": 0.5555555555555556, + "f1": 0.3846153846153847, + "f1_macro": 0.6923076923076923, + "f1_micro": 0.5555555555555556, + "f1_weighted": 0.7264957264957266 + }, + "duration": 8.288825988769531, + "num_run": 15, + "train_loss": { + "accuracy": 0.546242774566474, + "balanced_accuracy": 0.5, + "roc_auc": 0.4537121288713646, + "average_precision": 0.5218043063878082, + "log_loss": 0.7198673617633092, + "precision": 0.546242774566474, + "precision_macro": 0.773121387283237, + "precision_micro": 0.546242774566474, + "precision_weighted": 0.7941043803668683, + "recall": 0.0, + "recall_macro": 0.5, + "recall_micro": 0.546242774566474, + "recall_weighted": 0.546242774566474, + "f1": 0.3757455268389662, + "f1_macro": 0.6878727634194831, + "f1_micro": 0.546242774566474, + "f1_weighted": 0.7167400222939817 + }, + "test_loss": { + "accuracy": 0.5722543352601156, + "balanced_accuracy": 0.5, + "roc_auc": 0.49529074529074535, + "average_precision": 0.5699331879028938, + "log_loss": 0.7280194180549224, + "precision": 0.5722543352601156, + "precision_macro": 0.7861271676300579, + "precision_micro": 0.5722543352601156, + "precision_weighted": 0.8170336462962344, + "recall": 0.0, + "recall_macro": 0.5, + "recall_micro": 0.5722543352601156, + "recall_weighted": 0.5722543352601156, + "f1": 0.4008097165991902, + "f1_macro": 0.7004048582995951, + "f1_micro": 0.5722543352601157, + "f1_weighted": 0.7436989539210408 + }, + "configuration_origin": "Random Search (sorted)" + } + ] + ], + [ + [ + 9, + "{\"task_id\": \"Australian\"}", + 0, + 5.555555555555555 + ], + [ + 0.4444444444444444, + 5.485020637512207, + { + "__enum__": "StatusType.SUCCESS" + }, + 1637342743.4267018, + 1637342749.9442234, + { + "opt_loss": { + "accuracy": 0.4444444444444444, + "balanced_accuracy": 0.5, + "roc_auc": 0.5, + "average_precision": 0.5555555555555556, + "log_loss": 15.350567287868923, + "precision": 1.0, + "precision_macro": 0.7222222222222222, + "precision_micro": 0.4444444444444444, + "precision_weighted": 0.691358024691358, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.4444444444444444, + "recall_weighted": 0.4444444444444444, + "f1": 1.0, + "f1_macro": 0.6428571428571428, + "f1_micro": 0.4444444444444444, + "f1_weighted": 0.6031746031746031 + }, + "duration": 5.376826286315918, + "num_run": 16, + "train_loss": { + "accuracy": 0.45375722543352603, + "balanced_accuracy": 0.5, + "roc_auc": 0.5, + "average_precision": 0.546242774566474, + "log_loss": 15.67221934809161, + "precision": 1.0, + "precision_macro": 0.726878612716763, + "precision_micro": 0.45375722543352603, + "precision_weighted": 0.7016188312339203, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.45375722543352603, + "recall_weighted": 0.45375722543352603, + "f1": 1.0, + "f1_macro": 0.6467289719626168, + "f1_micro": 0.45375722543352603, + "f1_weighted": 0.6140565069418183 + }, + "test_loss": { + "accuracy": 0.4277456647398844, + "balanced_accuracy": 0.5, + "roc_auc": 0.5, + "average_precision": 0.5722543352601156, + "log_loss": 14.773811869538589, + "precision": 1.0, + "precision_macro": 0.7138728323699421, + "precision_micro": 0.4277456647398844, + "precision_weighted": 0.6725249757760032, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.4277456647398844, + "recall_weighted": 0.4277456647398844, + "f1": 1.0, + "f1_macro": 0.6360294117647058, + "f1_micro": 0.4277456647398844, + "f1_weighted": 0.5834325059503571 + }, + "configuration_origin": "Random Search" + } + ] + ], + [ + [ + 1, + "{\"task_id\": \"Australian\"}", + 0, + 16.666666666666664 + ], + [ + 0.15204678362573099, + 11.514830589294434, + { + "__enum__": "StatusType.SUCCESS" + }, + 1637342750.0053334, + 1637342762.5487585, + { + "opt_loss": { + "accuracy": 0.15204678362573099, + "balanced_accuracy": 0.15263157894736845, + "roc_auc": 0.08981994459833786, + "average_precision": 0.1040861796433199, + "log_loss": 0.5765479137672738, + "precision": 0.17948717948717952, + "precision_macro": 0.15425971877584788, + "precision_micro": 0.15204678362573099, + "precision_weighted": 0.15145666758569976, + "recall": 0.1578947368421053, + "recall_macro": 0.15263157894736845, + "recall_micro": 0.15204678362573099, + "recall_weighted": 0.15204678362573099, + "f1": 0.16883116883116878, + "f1_macro": 0.15356452058579717, + "f1_micro": 0.15204678362573099, + "f1_weighted": 0.15186822633631147 + }, + "duration": 11.44463586807251, + "num_run": 8, + "train_loss": { + "accuracy": 0.09537572254335258, + "balanced_accuracy": 0.10239948774980623, + "roc_auc": 0.03963198867657458, + "average_precision": 0.044469547423341305, + "log_loss": 0.28008669264774966, + "precision": 0.03731343283582089, + "precision_macro": 0.08469445226696704, + "precision_micro": 0.09537572254335258, + "precision_weighted": 0.0890765118675354, + "recall": 0.17834394904458595, + "recall_macro": 0.10239948774980623, + "recall_micro": 0.09537572254335258, + "recall_weighted": 0.09537572254335258, + "f1": 0.11340206185567014, + "f1_macro": 0.09784816309741107, + "f1_micro": 0.09537572254335258, + "f1_weighted": 0.0964096522295953 + }, + "test_loss": { + "accuracy": 0.1445086705202312, + "balanced_accuracy": 0.15356265356265353, + "roc_auc": 0.07821457821457822, + "average_precision": 0.08337624010373501, + "log_loss": 0.4291606295042948, + "precision": 0.13432835820895528, + "precision_macro": 0.14263587721768523, + "precision_micro": 0.1445086705202312, + "precision_weighted": 0.14383638574495827, + "recall": 0.21621621621621623, + "recall_macro": 0.15356265356265353, + "recall_micro": 0.1445086705202312, + "recall_weighted": 0.1445086705202312, + "f1": 0.17730496453900713, + "f1_macro": 0.14962809202560112, + "f1_micro": 0.1445086705202313, + "f1_weighted": 0.14562854397453084 + }, + "configuration_origin": "Default" + } + ] + ], + [ + [ + 1, + "{\"task_id\": \"Australian\"}", + 0, + 50.0 + ], + [ + 0.15204678362573099, + 15.370736837387085, + { + "__enum__": "StatusType.SUCCESS" + }, + 1637342762.794756, + 1637342779.192385, + { + "opt_loss": { + "accuracy": 0.15204678362573099, + "balanced_accuracy": 0.15263157894736845, + "roc_auc": 0.08981994459833786, + "average_precision": 0.1040861796433199, + "log_loss": 0.5765479137672738, + "precision": 0.17948717948717952, + "precision_macro": 0.15425971877584788, + "precision_micro": 0.15204678362573099, + "precision_weighted": 0.15145666758569976, + "recall": 0.1578947368421053, + "recall_macro": 0.15263157894736845, + "recall_micro": 0.15204678362573099, + "recall_weighted": 0.15204678362573099, + "f1": 0.16883116883116878, + "f1_macro": 0.15356452058579717, + "f1_micro": 0.15204678362573099, + "f1_weighted": 0.15186822633631147 + }, + "duration": 15.300711154937744, + "num_run": 8, + "train_loss": { + "accuracy": 0.09537572254335258, + "balanced_accuracy": 0.10239948774980623, + "roc_auc": 0.03963198867657458, + "average_precision": 0.044469547423341305, + "log_loss": 0.28008669264774966, + "precision": 0.03731343283582089, + "precision_macro": 0.08469445226696704, + "precision_micro": 0.09537572254335258, + "precision_weighted": 0.0890765118675354, + "recall": 0.17834394904458595, + "recall_macro": 0.10239948774980623, + "recall_micro": 0.09537572254335258, + "recall_weighted": 0.09537572254335258, + "f1": 0.11340206185567014, + "f1_macro": 0.09784816309741107, + "f1_micro": 0.09537572254335258, + "f1_weighted": 0.0964096522295953 + }, + "test_loss": { + "accuracy": 0.1445086705202312, + "balanced_accuracy": 0.15356265356265353, + "roc_auc": 0.07821457821457822, + "average_precision": 0.08337624010373501, + "log_loss": 0.4291606295042948, + "precision": 0.13432835820895528, + "precision_macro": 0.14263587721768523, + "precision_micro": 0.1445086705202312, + "precision_weighted": 0.14383638574495827, + "recall": 0.21621621621621623, + "recall_macro": 0.15356265356265353, + "recall_micro": 0.1445086705202312, + "recall_weighted": 0.1445086705202312, + "f1": 0.17730496453900713, + "f1_macro": 0.14962809202560112, + "f1_micro": 0.1445086705202313, + "f1_weighted": 0.14562854397453084 + }, + "configuration_origin": "Default" + } + ] + ], + [ + [ + 10, + "{\"task_id\": \"Australian\"}", + 0, + 16.666666666666664 + ], + [ + 0.4035087719298246, + 23.846530199050903, + { + "__enum__": "StatusType.SUCCESS" + }, + 1637342779.4572933, + 1637342804.3368232, + { + "opt_loss": { + "accuracy": 0.4035087719298246, + "balanced_accuracy": 0.39473684210526316, + "roc_auc": 0.3946675900277007, + "average_precision": 0.4846825737029168, + "log_loss": 6.419999084913276, + "precision": 0.4639175257731959, + "precision_macro": 0.39412092504876006, + "precision_micro": 0.4035087719298246, + "precision_weighted": 0.38636574719048944, + "recall": 0.3157894736842105, + "recall_macro": 0.39473684210526316, + "recall_micro": 0.4035087719298246, + "recall_weighted": 0.4035087719298246, + "f1": 0.3988439306358381, + "f1_macro": 0.4035639771522386, + "f1_micro": 0.4035087719298246, + "f1_weighted": 0.404088426765172 + }, + "duration": 23.588075160980225, + "num_run": 17, + "train_loss": { + "accuracy": 0.3988439306358381, + "balanced_accuracy": 0.3947359552455094, + "roc_auc": 0.4153776160145586, + "average_precision": 0.49525391358194226, + "log_loss": 7.800554750687936, + "precision": 0.4486486486486486, + "precision_macro": 0.39513177774047337, + "precision_micro": 0.3988439306358381, + "precision_weighted": 0.39018224054665374, + "recall": 0.35031847133757965, + "recall_macro": 0.3947359552455094, + "recall_micro": 0.3988439306358381, + "recall_weighted": 0.3988439306358381, + "f1": 0.4035087719298246, + "f1_macro": 0.39889724310776953, + "f1_micro": 0.3988439306358381, + "f1_weighted": 0.39847074333231924 + }, + "test_loss": { + "accuracy": 0.49132947976878616, + "balanced_accuracy": 0.4907179907179907, + "roc_auc": 0.45475020475020467, + "average_precision": 0.5080371591993963, + "log_loss": 8.656588848591884, + "precision": 0.5632183908045977, + "precision_macro": 0.49091152098369417, + "precision_micro": 0.49132947976878616, + "precision_weighted": 0.48046255135639593, + "recall": 0.4864864864864865, + "recall_macro": 0.4907179907179907, + "recall_micro": 0.49132947976878616, + "recall_weighted": 0.49132947976878616, + "f1": 0.5279503105590062, + "f1_macro": 0.4937048850092328, + "f1_micro": 0.49132947976878616, + "f1_weighted": 0.4887561240916355 + }, + "configuration_origin": "Random Search" + } + ] + ], + [ + [ + 11, + "{\"task_id\": \"Australian\"}", + 0, + 16.666666666666664 + ], + [ + 0.4444444444444444, + 6.757539510726929, + { + "__enum__": "StatusType.SUCCESS" + }, + 1637342813.02129, + 1637342820.8067145, + { + "opt_loss": { + "accuracy": 0.4444444444444444, + "balanced_accuracy": 0.5, + "roc_auc": 0.6250692520775624, + "average_precision": 0.642743659212315, + "log_loss": 0.6874508627674036, + "precision": 1.0, + "precision_macro": 0.7222222222222222, + "precision_micro": 0.4444444444444444, + "precision_weighted": 0.691358024691358, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.4444444444444444, + "recall_weighted": 0.4444444444444444, + "f1": 1.0, + "f1_macro": 0.6428571428571428, + "f1_micro": 0.4444444444444444, + "f1_weighted": 0.6031746031746031 + }, + "duration": 6.695321321487427, + "num_run": 18, + "train_loss": { + "accuracy": 0.45375722543352603, + "balanced_accuracy": 0.5, + "roc_auc": 0.5560273649445624, + "average_precision": 0.5899410773859022, + "log_loss": 0.689653262926664, + "precision": 1.0, + "precision_macro": 0.726878612716763, + "precision_micro": 0.45375722543352603, + "precision_weighted": 0.7016188312339203, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.45375722543352603, + "recall_weighted": 0.45375722543352603, + "f1": 1.0, + "f1_macro": 0.6467289719626168, + "f1_micro": 0.45375722543352603, + "f1_weighted": 0.6140565069418183 + }, + "test_loss": { + "accuracy": 0.4277456647398844, + "balanced_accuracy": 0.5, + "roc_auc": 0.558968058968059, + "average_precision": 0.6279915052238905, + "log_loss": 0.6829987437049777, + "precision": 1.0, + "precision_macro": 0.7138728323699421, + "precision_micro": 0.4277456647398844, + "precision_weighted": 0.6725249757760032, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.4277456647398844, + "recall_weighted": 0.4277456647398844, + "f1": 1.0, + "f1_macro": 0.6360294117647058, + "f1_micro": 0.4277456647398844, + "f1_weighted": 0.5834325059503571 + }, + "configuration_origin": "Random Search (sorted)" + } + ] + ], + [ + [ + 12, + "{\"task_id\": \"Australian\"}", + 0, + 16.666666666666664 + ], + [ + 0.4444444444444444, + 15.061991930007935, + { + "__enum__": "StatusType.SUCCESS" + }, + 1637342829.9214745, + 1637342846.0210106, + { + "opt_loss": { + "accuracy": 0.4444444444444444, + "balanced_accuracy": 0.5, + "roc_auc": 0.378393351800554, + "average_precision": 0.4680399341300143, + "log_loss": 0.6910723817278768, + "precision": 1.0, + "precision_macro": 0.7222222222222222, + "precision_micro": 0.4444444444444444, + "precision_weighted": 0.691358024691358, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.4444444444444444, + "recall_weighted": 0.4444444444444444, + "f1": 1.0, + "f1_macro": 0.6428571428571428, + "f1_micro": 0.4444444444444444, + "f1_weighted": 0.6031746031746031 + }, + "duration": 14.850486516952515, + "num_run": 19, + "train_loss": { + "accuracy": 0.45375722543352603, + "balanced_accuracy": 0.5, + "roc_auc": 0.44353452633707413, + "average_precision": 0.4796876232765652, + "log_loss": 0.6915661034556483, + "precision": 1.0, + "precision_macro": 0.726878612716763, + "precision_micro": 0.45375722543352603, + "precision_weighted": 0.7016188312339203, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.45375722543352603, + "recall_weighted": 0.45375722543352603, + "f1": 1.0, + "f1_macro": 0.6467289719626168, + "f1_micro": 0.45375722543352603, + "f1_weighted": 0.6140565069418183 + }, + "test_loss": { + "accuracy": 0.4277456647398844, + "balanced_accuracy": 0.5, + "roc_auc": 0.4836199836199836, + "average_precision": 0.5850466880722343, + "log_loss": 0.6912664895112803, + "precision": 1.0, + "precision_macro": 0.7138728323699421, + "precision_micro": 0.4277456647398844, + "precision_weighted": 0.6725249757760032, + "recall": 1.0, + "recall_macro": 0.5, + "recall_micro": 0.4277456647398844, + "recall_weighted": 0.4277456647398844, + "f1": 1.0, + "f1_macro": 0.6360294117647058, + "f1_micro": 0.4277456647398844, + "f1_weighted": 0.5834325059503571 + }, + "configuration_origin": "Random Search (sorted)" + } + ] + ], + [ + [ + 10, + "{\"task_id\": \"Australian\"}", + 0, + 50.0 + ], + [ + 1.0, + 50.010520696640015, + { + "__enum__": "StatusType.TIMEOUT" + }, + 1637342846.0745292, + 1637342897.1205413, + { + "error": "Timeout", + "configuration_origin": "Random Search" + } + ] + ], + [ + [ + 13, + "{\"task_id\": \"Australian\"}", + 0, + 50.0 + ], + [ + 1.0, + 22.011935234069824, + { + "__enum__": "StatusType.TIMEOUT" + }, + 1637342905.7068844, + 1637342928.7456856, + { + "error": "Timeout", + "configuration_origin": "Random Search (sorted)" + } + ] + ], + [ + [ + 14, + "{\"task_id\": \"Australian\"}", + 0, + 50.0 + ], + [ + 1.0, + 0.0, + { + "__enum__": "StatusType.STOP" + }, + 1637342928.8133125, + 1637342928.8133128, + {} + ] + ] + ], + "config_origins": { + "1": "Default", + "2": "Random Search", + "3": "Random Search (sorted)", + "4": "Random Search", + "5": "Random Search", + "6": "Random Search", + "7": "Random Search (sorted)", + "8": "Random Search (sorted)", + "9": "Random Search", + "10": "Random Search", + "11": "Random Search (sorted)", + "12": "Random Search (sorted)", + "13": "Random Search (sorted)", + "14": "Random Search" + }, + "configs": { + "1": { + "data_loader:batch_size": 64, + "encoder:__choice__": "OneHotEncoder", + "feature_preprocessor:__choice__": "NoFeaturePreprocessor", + "imputer:categorical_strategy": "most_frequent", + "imputer:numerical_strategy": "mean", + "lr_scheduler:__choice__": "ReduceLROnPlateau", + "network_backbone:__choice__": "ShapedMLPBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "XavierInit", + "optimizer:__choice__": "AdamOptimizer", + "scaler:__choice__": "StandardScaler", + "trainer:__choice__": "StandardTrainer", + "lr_scheduler:ReduceLROnPlateau:factor": 0.1, + "lr_scheduler:ReduceLROnPlateau:mode": "min", + "lr_scheduler:ReduceLROnPlateau:patience": 10, + "network_backbone:ShapedMLPBackbone:activation": "relu", + "network_backbone:ShapedMLPBackbone:max_units": 200, + "network_backbone:ShapedMLPBackbone:mlp_shape": "funnel", + "network_backbone:ShapedMLPBackbone:num_groups": 5, + "network_backbone:ShapedMLPBackbone:output_dim": 200, + "network_backbone:ShapedMLPBackbone:use_dropout": false, + "network_head:fully_connected:num_layers": 2, + "network_init:XavierInit:bias_strategy": "Normal", + "optimizer:AdamOptimizer:beta1": 0.9, + "optimizer:AdamOptimizer:beta2": 0.9, + "optimizer:AdamOptimizer:lr": 0.01, + "optimizer:AdamOptimizer:weight_decay": 0.0, + "trainer:StandardTrainer:weighted_loss": true, + "network_head:fully_connected:activation": "relu", + "network_head:fully_connected:units_layer_1": 128 + }, + "2": { + "data_loader:batch_size": 142, + "encoder:__choice__": "NoEncoder", + "feature_preprocessor:__choice__": "PowerTransformer", + "imputer:categorical_strategy": "constant_!missing!", + "imputer:numerical_strategy": "median", + "lr_scheduler:__choice__": "NoScheduler", + "network_backbone:__choice__": "ShapedResNetBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "KaimingInit", + "optimizer:__choice__": "AdamOptimizer", + "scaler:__choice__": "Normalizer", + "trainer:__choice__": "MixUpTrainer", + "feature_preprocessor:PowerTransformer:standardize": true, + "network_backbone:ShapedResNetBackbone:activation": "relu", + "network_backbone:ShapedResNetBackbone:blocks_per_group": 1, + "network_backbone:ShapedResNetBackbone:max_units": 175, + "network_backbone:ShapedResNetBackbone:num_groups": 3, + "network_backbone:ShapedResNetBackbone:output_dim": 550, + "network_backbone:ShapedResNetBackbone:resnet_shape": "funnel", + "network_backbone:ShapedResNetBackbone:use_dropout": false, + "network_backbone:ShapedResNetBackbone:use_shake_drop": true, + "network_backbone:ShapedResNetBackbone:use_shake_shake": true, + "network_head:fully_connected:num_layers": 2, + "network_init:KaimingInit:bias_strategy": "Normal", + "optimizer:AdamOptimizer:beta1": 0.8660298289969375, + "optimizer:AdamOptimizer:beta2": 0.9517157453274235, + "optimizer:AdamOptimizer:lr": 1.0377748473731365e-05, + "optimizer:AdamOptimizer:weight_decay": 0.07437634123996516, + "scaler:Normalizer:norm": "mean_abs", + "trainer:MixUpTrainer:alpha": 0.13179357367568267, + "trainer:MixUpTrainer:weighted_loss": true, + "network_backbone:ShapedResNetBackbone:max_shake_drop_probability": 0.7993610769045779, + "network_head:fully_connected:activation": "sigmoid", + "network_head:fully_connected:units_layer_1": 308 + }, + "3": { + "data_loader:batch_size": 246, + "encoder:__choice__": "OneHotEncoder", + "feature_preprocessor:__choice__": "PowerTransformer", + "imputer:categorical_strategy": "constant_!missing!", + "imputer:numerical_strategy": "most_frequent", + "lr_scheduler:__choice__": "CosineAnnealingWarmRestarts", + "network_backbone:__choice__": "ResNetBackbone", + "network_embedding:__choice__": "LearnedEntityEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "XavierInit", + "optimizer:__choice__": "AdamOptimizer", + "scaler:__choice__": "Normalizer", + "trainer:__choice__": "StandardTrainer", + "feature_preprocessor:PowerTransformer:standardize": true, + "lr_scheduler:CosineAnnealingWarmRestarts:T_0": 17, + "lr_scheduler:CosineAnnealingWarmRestarts:T_mult": 1.0577034671447638, + "network_backbone:ResNetBackbone:activation": "sigmoid", + "network_backbone:ResNetBackbone:blocks_per_group_0": 1, + "network_backbone:ResNetBackbone:blocks_per_group_1": 4, + "network_backbone:ResNetBackbone:num_groups": 10, + "network_backbone:ResNetBackbone:num_units_0": 974, + "network_backbone:ResNetBackbone:num_units_1": 151, + "network_backbone:ResNetBackbone:use_dropout": true, + "network_backbone:ResNetBackbone:use_shake_drop": true, + "network_backbone:ResNetBackbone:use_shake_shake": true, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_0": 0.6898659803969109, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_1": 0.6193894885012183, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_2": 0.27044405840757246, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_3": 0.3353276257116905, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_4": 0.25330009522745545, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_5": 0.28087428370045076, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_6": 0.985667346693578, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_7": 0.532995443030165, + "network_embedding:LearnedEntityEmbedding:min_unique_values_for_embedding": 4, + "network_head:fully_connected:num_layers": 4, + "network_init:XavierInit:bias_strategy": "Normal", + "optimizer:AdamOptimizer:beta1": 0.9319239463981089, + "optimizer:AdamOptimizer:beta2": 0.9290660642046109, + "optimizer:AdamOptimizer:lr": 4.934398361769327e-05, + "optimizer:AdamOptimizer:weight_decay": 0.0647885374302594, + "scaler:Normalizer:norm": "mean_abs", + "trainer:StandardTrainer:weighted_loss": true, + "network_backbone:ResNetBackbone:blocks_per_group_10": 2, + "network_backbone:ResNetBackbone:blocks_per_group_2": 1, + "network_backbone:ResNetBackbone:blocks_per_group_3": 1, + "network_backbone:ResNetBackbone:blocks_per_group_4": 3, + "network_backbone:ResNetBackbone:blocks_per_group_5": 1, + "network_backbone:ResNetBackbone:blocks_per_group_6": 3, + "network_backbone:ResNetBackbone:blocks_per_group_7": 4, + "network_backbone:ResNetBackbone:blocks_per_group_8": 1, + "network_backbone:ResNetBackbone:blocks_per_group_9": 3, + "network_backbone:ResNetBackbone:dropout_0": 0.1998483800982469, + "network_backbone:ResNetBackbone:dropout_1": 0.21671729531007777, + "network_backbone:ResNetBackbone:dropout_10": 0.2027668457966562, + "network_backbone:ResNetBackbone:dropout_2": 0.7140248388727144, + "network_backbone:ResNetBackbone:dropout_3": 0.1324478677866992, + "network_backbone:ResNetBackbone:dropout_4": 0.26711076053122573, + "network_backbone:ResNetBackbone:dropout_5": 0.2895993889716623, + "network_backbone:ResNetBackbone:dropout_6": 0.047419135928320616, + "network_backbone:ResNetBackbone:dropout_7": 0.593522761474697, + "network_backbone:ResNetBackbone:dropout_8": 0.11825542268484464, + "network_backbone:ResNetBackbone:dropout_9": 0.5802180655508312, + "network_backbone:ResNetBackbone:max_shake_drop_probability": 0.8422119101175598, + "network_backbone:ResNetBackbone:num_units_10": 1012, + "network_backbone:ResNetBackbone:num_units_2": 793, + "network_backbone:ResNetBackbone:num_units_3": 184, + "network_backbone:ResNetBackbone:num_units_4": 1022, + "network_backbone:ResNetBackbone:num_units_5": 88, + "network_backbone:ResNetBackbone:num_units_6": 666, + "network_backbone:ResNetBackbone:num_units_7": 927, + "network_backbone:ResNetBackbone:num_units_8": 614, + "network_backbone:ResNetBackbone:num_units_9": 552, + "network_head:fully_connected:activation": "relu", + "network_head:fully_connected:units_layer_1": 92, + "network_head:fully_connected:units_layer_2": 202, + "network_head:fully_connected:units_layer_3": 171 + }, + "4": { + "data_loader:batch_size": 269, + "encoder:__choice__": "OneHotEncoder", + "feature_preprocessor:__choice__": "PowerTransformer", + "imputer:categorical_strategy": "constant_!missing!", + "imputer:numerical_strategy": "median", + "lr_scheduler:__choice__": "CosineAnnealingLR", + "network_backbone:__choice__": "ShapedMLPBackbone", + "network_embedding:__choice__": "LearnedEntityEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "KaimingInit", + "optimizer:__choice__": "RMSpropOptimizer", + "scaler:__choice__": "MinMaxScaler", + "trainer:__choice__": "StandardTrainer", + "feature_preprocessor:PowerTransformer:standardize": true, + "lr_scheduler:CosineAnnealingLR:T_max": 57, + "network_backbone:ShapedMLPBackbone:activation": "relu", + "network_backbone:ShapedMLPBackbone:max_units": 199, + "network_backbone:ShapedMLPBackbone:mlp_shape": "stairs", + "network_backbone:ShapedMLPBackbone:num_groups": 12, + "network_backbone:ShapedMLPBackbone:output_dim": 641, + "network_backbone:ShapedMLPBackbone:use_dropout": true, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_0": 0.8093046015402414, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_1": 0.6952888698136637, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_2": 0.7136167420874352, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_3": 0.7071870846094686, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_4": 0.8821351885181623, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_5": 0.21840740866837938, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_6": 0.7366390825638998, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_7": 0.548715467945816, + "network_embedding:LearnedEntityEmbedding:min_unique_values_for_embedding": 7, + "network_head:fully_connected:num_layers": 3, + "network_init:KaimingInit:bias_strategy": "Zero", + "optimizer:RMSpropOptimizer:alpha": 0.23716801972855298, + "optimizer:RMSpropOptimizer:lr": 0.0011708542709120838, + "optimizer:RMSpropOptimizer:momentum": 0.5620565618493047, + "optimizer:RMSpropOptimizer:weight_decay": 0.05858239202799009, + "trainer:StandardTrainer:weighted_loss": true, + "network_backbone:ShapedMLPBackbone:max_dropout": 0.20819857031346878, + "network_head:fully_connected:activation": "relu", + "network_head:fully_connected:units_layer_1": 177, + "network_head:fully_connected:units_layer_2": 196 + }, + "5": { + "data_loader:batch_size": 191, + "encoder:__choice__": "OneHotEncoder", + "feature_preprocessor:__choice__": "RandomKitchenSinks", + "imputer:categorical_strategy": "constant_!missing!", + "imputer:numerical_strategy": "most_frequent", + "lr_scheduler:__choice__": "ExponentialLR", + "network_backbone:__choice__": "ShapedResNetBackbone", + "network_embedding:__choice__": "LearnedEntityEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "SparseInit", + "optimizer:__choice__": "AdamOptimizer", + "scaler:__choice__": "MinMaxScaler", + "trainer:__choice__": "MixUpTrainer", + "feature_preprocessor:RandomKitchenSinks:gamma": 0.00023806069646323692, + "feature_preprocessor:RandomKitchenSinks:n_components": 6, + "lr_scheduler:ExponentialLR:gamma": 0.7718494018636944, + "network_backbone:ShapedResNetBackbone:activation": "tanh", + "network_backbone:ShapedResNetBackbone:blocks_per_group": 3, + "network_backbone:ShapedResNetBackbone:max_units": 869, + "network_backbone:ShapedResNetBackbone:num_groups": 2, + "network_backbone:ShapedResNetBackbone:output_dim": 868, + "network_backbone:ShapedResNetBackbone:resnet_shape": "triangle", + "network_backbone:ShapedResNetBackbone:use_dropout": true, + "network_backbone:ShapedResNetBackbone:use_shake_drop": true, + "network_backbone:ShapedResNetBackbone:use_shake_shake": true, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_0": 0.08846693746970624, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_1": 0.6597252449477167, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_2": 0.11290616066859738, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_3": 0.4187266624427779, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_4": 0.026810815995375492, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_5": 0.02021466731982824, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_6": 0.01616376260397212, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_7": 0.6463510235731745, + "network_embedding:LearnedEntityEmbedding:min_unique_values_for_embedding": 6, + "network_head:fully_connected:num_layers": 3, + "network_init:SparseInit:bias_strategy": "Normal", + "optimizer:AdamOptimizer:beta1": 0.9966044931531224, + "optimizer:AdamOptimizer:beta2": 0.9293356180290759, + "optimizer:AdamOptimizer:lr": 0.07180366191531826, + "optimizer:AdamOptimizer:weight_decay": 0.012304534471441598, + "trainer:MixUpTrainer:alpha": 0.8900376828213522, + "trainer:MixUpTrainer:weighted_loss": true, + "network_backbone:ShapedResNetBackbone:max_dropout": 0.6688622458251051, + "network_backbone:ShapedResNetBackbone:max_shake_drop_probability": 0.28903761225065516, + "network_head:fully_connected:activation": "sigmoid", + "network_head:fully_connected:units_layer_1": 198, + "network_head:fully_connected:units_layer_2": 283 + }, + "6": { + "data_loader:batch_size": 53, + "encoder:__choice__": "OneHotEncoder", + "feature_preprocessor:__choice__": "PowerTransformer", + "imputer:categorical_strategy": "constant_!missing!", + "imputer:numerical_strategy": "median", + "lr_scheduler:__choice__": "StepLR", + "network_backbone:__choice__": "ResNetBackbone", + "network_embedding:__choice__": "LearnedEntityEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "OrthogonalInit", + "optimizer:__choice__": "AdamOptimizer", + "scaler:__choice__": "Normalizer", + "trainer:__choice__": "MixUpTrainer", + "feature_preprocessor:PowerTransformer:standardize": false, + "lr_scheduler:StepLR:gamma": 0.27529217359012764, + "lr_scheduler:StepLR:step_size": 8, + "network_backbone:ResNetBackbone:activation": "sigmoid", + "network_backbone:ResNetBackbone:blocks_per_group_0": 3, + "network_backbone:ResNetBackbone:blocks_per_group_1": 1, + "network_backbone:ResNetBackbone:num_groups": 6, + "network_backbone:ResNetBackbone:num_units_0": 884, + "network_backbone:ResNetBackbone:num_units_1": 160, + "network_backbone:ResNetBackbone:use_dropout": false, + "network_backbone:ResNetBackbone:use_shake_drop": true, + "network_backbone:ResNetBackbone:use_shake_shake": false, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_0": 0.5659324295712268, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_1": 0.8744001957677244, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_2": 0.3415903412295024, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_3": 0.8599314829187148, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_4": 0.9869678384973877, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_5": 0.7490528427155283, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_6": 0.9979477892240094, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_7": 0.4171316119626819, + "network_embedding:LearnedEntityEmbedding:min_unique_values_for_embedding": 5, + "network_head:fully_connected:num_layers": 1, + "network_init:OrthogonalInit:bias_strategy": "Zero", + "optimizer:AdamOptimizer:beta1": 0.9576776568384536, + "optimizer:AdamOptimizer:beta2": 0.9605074039230137, + "optimizer:AdamOptimizer:lr": 0.02098507521065345, + "optimizer:AdamOptimizer:weight_decay": 0.021686007599294888, + "scaler:Normalizer:norm": "mean_abs", + "trainer:MixUpTrainer:alpha": 0.8399712211486785, + "trainer:MixUpTrainer:weighted_loss": true, + "network_backbone:ResNetBackbone:blocks_per_group_2": 3, + "network_backbone:ResNetBackbone:blocks_per_group_3": 3, + "network_backbone:ResNetBackbone:blocks_per_group_4": 1, + "network_backbone:ResNetBackbone:blocks_per_group_5": 1, + "network_backbone:ResNetBackbone:blocks_per_group_6": 3, + "network_backbone:ResNetBackbone:max_shake_drop_probability": 0.09160627667494659, + "network_backbone:ResNetBackbone:num_units_2": 396, + "network_backbone:ResNetBackbone:num_units_3": 587, + "network_backbone:ResNetBackbone:num_units_4": 169, + "network_backbone:ResNetBackbone:num_units_5": 546, + "network_backbone:ResNetBackbone:num_units_6": 92 + }, + "7": { + "data_loader:batch_size": 232, + "encoder:__choice__": "NoEncoder", + "feature_preprocessor:__choice__": "RandomKitchenSinks", + "imputer:categorical_strategy": "most_frequent", + "imputer:numerical_strategy": "most_frequent", + "lr_scheduler:__choice__": "NoScheduler", + "network_backbone:__choice__": "ResNetBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "XavierInit", + "optimizer:__choice__": "AdamOptimizer", + "scaler:__choice__": "Normalizer", + "trainer:__choice__": "StandardTrainer", + "feature_preprocessor:RandomKitchenSinks:gamma": 5.7968061542283495e-05, + "feature_preprocessor:RandomKitchenSinks:n_components": 5, + "network_backbone:ResNetBackbone:activation": "sigmoid", + "network_backbone:ResNetBackbone:blocks_per_group_0": 2, + "network_backbone:ResNetBackbone:blocks_per_group_1": 3, + "network_backbone:ResNetBackbone:num_groups": 14, + "network_backbone:ResNetBackbone:num_units_0": 63, + "network_backbone:ResNetBackbone:num_units_1": 229, + "network_backbone:ResNetBackbone:use_dropout": true, + "network_backbone:ResNetBackbone:use_shake_drop": false, + "network_backbone:ResNetBackbone:use_shake_shake": true, + "network_head:fully_connected:num_layers": 2, + "network_init:XavierInit:bias_strategy": "Normal", + "optimizer:AdamOptimizer:beta1": 0.9646917651093316, + "optimizer:AdamOptimizer:beta2": 0.9949552394978046, + "optimizer:AdamOptimizer:lr": 0.018422761006289576, + "optimizer:AdamOptimizer:weight_decay": 0.01700341747601285, + "scaler:Normalizer:norm": "mean_squared", + "trainer:StandardTrainer:weighted_loss": false, + "network_backbone:ResNetBackbone:blocks_per_group_10": 4, + "network_backbone:ResNetBackbone:blocks_per_group_11": 3, + "network_backbone:ResNetBackbone:blocks_per_group_12": 2, + "network_backbone:ResNetBackbone:blocks_per_group_13": 1, + "network_backbone:ResNetBackbone:blocks_per_group_14": 3, + "network_backbone:ResNetBackbone:blocks_per_group_2": 1, + "network_backbone:ResNetBackbone:blocks_per_group_3": 3, + "network_backbone:ResNetBackbone:blocks_per_group_4": 4, + "network_backbone:ResNetBackbone:blocks_per_group_5": 3, + "network_backbone:ResNetBackbone:blocks_per_group_6": 4, + "network_backbone:ResNetBackbone:blocks_per_group_7": 2, + "network_backbone:ResNetBackbone:blocks_per_group_8": 3, + "network_backbone:ResNetBackbone:blocks_per_group_9": 2, + "network_backbone:ResNetBackbone:dropout_0": 0.3872694110962167, + "network_backbone:ResNetBackbone:dropout_1": 0.7182095865182352, + "network_backbone:ResNetBackbone:dropout_10": 0.7518775284870586, + "network_backbone:ResNetBackbone:dropout_11": 0.3717581189860213, + "network_backbone:ResNetBackbone:dropout_12": 0.055178370982331075, + "network_backbone:ResNetBackbone:dropout_13": 0.5670307132839905, + "network_backbone:ResNetBackbone:dropout_14": 0.7859566818562779, + "network_backbone:ResNetBackbone:dropout_2": 0.5796670187291707, + "network_backbone:ResNetBackbone:dropout_3": 0.05370643307213783, + "network_backbone:ResNetBackbone:dropout_4": 0.37288408223729974, + "network_backbone:ResNetBackbone:dropout_5": 0.47179695650262793, + "network_backbone:ResNetBackbone:dropout_6": 0.20070003914010803, + "network_backbone:ResNetBackbone:dropout_7": 0.638048407623313, + "network_backbone:ResNetBackbone:dropout_8": 0.6190670404279601, + "network_backbone:ResNetBackbone:dropout_9": 0.33325853682297146, + "network_backbone:ResNetBackbone:num_units_10": 925, + "network_backbone:ResNetBackbone:num_units_11": 164, + "network_backbone:ResNetBackbone:num_units_12": 247, + "network_backbone:ResNetBackbone:num_units_13": 339, + "network_backbone:ResNetBackbone:num_units_14": 769, + "network_backbone:ResNetBackbone:num_units_2": 502, + "network_backbone:ResNetBackbone:num_units_3": 101, + "network_backbone:ResNetBackbone:num_units_4": 842, + "network_backbone:ResNetBackbone:num_units_5": 906, + "network_backbone:ResNetBackbone:num_units_6": 933, + "network_backbone:ResNetBackbone:num_units_7": 329, + "network_backbone:ResNetBackbone:num_units_8": 898, + "network_backbone:ResNetBackbone:num_units_9": 161, + "network_head:fully_connected:activation": "relu", + "network_head:fully_connected:units_layer_1": 327 + }, + "8": { + "data_loader:batch_size": 164, + "encoder:__choice__": "NoEncoder", + "feature_preprocessor:__choice__": "NoFeaturePreprocessor", + "imputer:categorical_strategy": "most_frequent", + "imputer:numerical_strategy": "mean", + "lr_scheduler:__choice__": "StepLR", + "network_backbone:__choice__": "ShapedMLPBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "NoInit", + "optimizer:__choice__": "RMSpropOptimizer", + "scaler:__choice__": "MinMaxScaler", + "trainer:__choice__": "MixUpTrainer", + "lr_scheduler:StepLR:gamma": 0.2905213739360219, + "lr_scheduler:StepLR:step_size": 10, + "network_backbone:ShapedMLPBackbone:activation": "sigmoid", + "network_backbone:ShapedMLPBackbone:max_units": 903, + "network_backbone:ShapedMLPBackbone:mlp_shape": "brick", + "network_backbone:ShapedMLPBackbone:num_groups": 10, + "network_backbone:ShapedMLPBackbone:output_dim": 943, + "network_backbone:ShapedMLPBackbone:use_dropout": false, + "network_head:fully_connected:num_layers": 3, + "network_init:NoInit:bias_strategy": "Zero", + "optimizer:RMSpropOptimizer:alpha": 0.25445785033325663, + "optimizer:RMSpropOptimizer:lr": 0.00012058949092384073, + "optimizer:RMSpropOptimizer:momentum": 0.6601732030357997, + "optimizer:RMSpropOptimizer:weight_decay": 0.030275825765581223, + "trainer:MixUpTrainer:alpha": 0.2222082093355312, + "trainer:MixUpTrainer:weighted_loss": true, + "network_head:fully_connected:activation": "sigmoid", + "network_head:fully_connected:units_layer_1": 110, + "network_head:fully_connected:units_layer_2": 70 + }, + "9": { + "data_loader:batch_size": 94, + "encoder:__choice__": "NoEncoder", + "feature_preprocessor:__choice__": "PolynomialFeatures", + "imputer:categorical_strategy": "most_frequent", + "imputer:numerical_strategy": "mean", + "lr_scheduler:__choice__": "CyclicLR", + "network_backbone:__choice__": "MLPBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "KaimingInit", + "optimizer:__choice__": "RMSpropOptimizer", + "scaler:__choice__": "StandardScaler", + "trainer:__choice__": "StandardTrainer", + "feature_preprocessor:PolynomialFeatures:degree": 2, + "feature_preprocessor:PolynomialFeatures:include_bias": false, + "feature_preprocessor:PolynomialFeatures:interaction_only": false, + "lr_scheduler:CyclicLR:base_lr": 0.09510483864725039, + "lr_scheduler:CyclicLR:max_lr": 0.026215723559513626, + "lr_scheduler:CyclicLR:mode": "triangular", + "lr_scheduler:CyclicLR:step_size_up": 3829, + "network_backbone:MLPBackbone:activation": "sigmoid", + "network_backbone:MLPBackbone:num_groups": 7, + "network_backbone:MLPBackbone:num_units_1": 47, + "network_backbone:MLPBackbone:use_dropout": true, + "network_head:fully_connected:num_layers": 3, + "network_init:KaimingInit:bias_strategy": "Zero", + "optimizer:RMSpropOptimizer:alpha": 0.75085811094601, + "optimizer:RMSpropOptimizer:lr": 0.0002950013672615944, + "optimizer:RMSpropOptimizer:momentum": 0.515129966307681, + "optimizer:RMSpropOptimizer:weight_decay": 0.01979731884468683, + "trainer:StandardTrainer:weighted_loss": true, + "network_backbone:MLPBackbone:dropout_1": 0.5362963908147109, + "network_backbone:MLPBackbone:dropout_2": 0.09403575191589564, + "network_backbone:MLPBackbone:dropout_3": 0.5576340928985162, + "network_backbone:MLPBackbone:dropout_4": 0.3102921398336836, + "network_backbone:MLPBackbone:dropout_5": 0.36841155269138837, + "network_backbone:MLPBackbone:dropout_6": 0.459182557172949, + "network_backbone:MLPBackbone:dropout_7": 0.2741849570242409, + "network_backbone:MLPBackbone:num_units_2": 323, + "network_backbone:MLPBackbone:num_units_3": 424, + "network_backbone:MLPBackbone:num_units_4": 637, + "network_backbone:MLPBackbone:num_units_5": 668, + "network_backbone:MLPBackbone:num_units_6": 507, + "network_backbone:MLPBackbone:num_units_7": 972, + "network_head:fully_connected:activation": "sigmoid", + "network_head:fully_connected:units_layer_1": 482, + "network_head:fully_connected:units_layer_2": 425 + }, + "10": { + "data_loader:batch_size": 70, + "encoder:__choice__": "OneHotEncoder", + "feature_preprocessor:__choice__": "PowerTransformer", + "imputer:categorical_strategy": "most_frequent", + "imputer:numerical_strategy": "constant_zero", + "lr_scheduler:__choice__": "CyclicLR", + "network_backbone:__choice__": "ShapedMLPBackbone", + "network_embedding:__choice__": "LearnedEntityEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "SparseInit", + "optimizer:__choice__": "RMSpropOptimizer", + "scaler:__choice__": "NoScaler", + "trainer:__choice__": "MixUpTrainer", + "feature_preprocessor:PowerTransformer:standardize": false, + "lr_scheduler:CyclicLR:base_lr": 0.05611929109855669, + "lr_scheduler:CyclicLR:max_lr": 0.01831055731936772, + "lr_scheduler:CyclicLR:mode": "triangular2", + "lr_scheduler:CyclicLR:step_size_up": 3104, + "network_backbone:ShapedMLPBackbone:activation": "tanh", + "network_backbone:ShapedMLPBackbone:max_units": 759, + "network_backbone:ShapedMLPBackbone:mlp_shape": "brick", + "network_backbone:ShapedMLPBackbone:num_groups": 10, + "network_backbone:ShapedMLPBackbone:output_dim": 155, + "network_backbone:ShapedMLPBackbone:use_dropout": true, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_0": 0.14333490052026576, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_1": 0.7794060644969828, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_2": 0.28020395999441905, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_3": 0.2820327943739419, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_4": 0.7390548552027222, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_5": 0.025302711343403672, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_6": 0.5677825375428477, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_7": 0.7093786601691139, + "network_embedding:LearnedEntityEmbedding:min_unique_values_for_embedding": 6, + "network_head:fully_connected:num_layers": 4, + "network_init:SparseInit:bias_strategy": "Zero", + "optimizer:RMSpropOptimizer:alpha": 0.15659144532965727, + "optimizer:RMSpropOptimizer:lr": 0.015691888676781927, + "optimizer:RMSpropOptimizer:momentum": 0.30317416976729206, + "optimizer:RMSpropOptimizer:weight_decay": 0.010642526008626797, + "trainer:MixUpTrainer:alpha": 0.3709089665342886, + "trainer:MixUpTrainer:weighted_loss": false, + "network_backbone:ShapedMLPBackbone:max_dropout": 0.3789762581174825, + "network_head:fully_connected:activation": "tanh", + "network_head:fully_connected:units_layer_1": 499, + "network_head:fully_connected:units_layer_2": 465, + "network_head:fully_connected:units_layer_3": 238 + }, + "11": { + "data_loader:batch_size": 274, + "encoder:__choice__": "NoEncoder", + "feature_preprocessor:__choice__": "RandomKitchenSinks", + "imputer:categorical_strategy": "constant_!missing!", + "imputer:numerical_strategy": "mean", + "lr_scheduler:__choice__": "CyclicLR", + "network_backbone:__choice__": "ShapedMLPBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "SparseInit", + "optimizer:__choice__": "AdamOptimizer", + "scaler:__choice__": "Normalizer", + "trainer:__choice__": "StandardTrainer", + "feature_preprocessor:RandomKitchenSinks:gamma": 0.00017836687895829377, + "feature_preprocessor:RandomKitchenSinks:n_components": 3, + "lr_scheduler:CyclicLR:base_lr": 0.061001847805883254, + "lr_scheduler:CyclicLR:max_lr": 0.037867703829357294, + "lr_scheduler:CyclicLR:mode": "triangular", + "lr_scheduler:CyclicLR:step_size_up": 3395, + "network_backbone:ShapedMLPBackbone:activation": "relu", + "network_backbone:ShapedMLPBackbone:max_units": 94, + "network_backbone:ShapedMLPBackbone:mlp_shape": "diamond", + "network_backbone:ShapedMLPBackbone:num_groups": 4, + "network_backbone:ShapedMLPBackbone:output_dim": 763, + "network_backbone:ShapedMLPBackbone:use_dropout": false, + "network_head:fully_connected:num_layers": 3, + "network_init:SparseInit:bias_strategy": "Zero", + "optimizer:AdamOptimizer:beta1": 0.9010241766841086, + "optimizer:AdamOptimizer:beta2": 0.9275862063741073, + "optimizer:AdamOptimizer:lr": 0.00048241454070108375, + "optimizer:AdamOptimizer:weight_decay": 0.058438892093437125, + "scaler:Normalizer:norm": "max", + "trainer:StandardTrainer:weighted_loss": true, + "network_head:fully_connected:activation": "relu", + "network_head:fully_connected:units_layer_1": 293, + "network_head:fully_connected:units_layer_2": 177 + }, + "12": { + "data_loader:batch_size": 191, + "encoder:__choice__": "NoEncoder", + "feature_preprocessor:__choice__": "NoFeaturePreprocessor", + "imputer:categorical_strategy": "constant_!missing!", + "imputer:numerical_strategy": "median", + "lr_scheduler:__choice__": "CosineAnnealingWarmRestarts", + "network_backbone:__choice__": "ResNetBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "XavierInit", + "optimizer:__choice__": "RMSpropOptimizer", + "scaler:__choice__": "Normalizer", + "trainer:__choice__": "StandardTrainer", + "lr_scheduler:CosineAnnealingWarmRestarts:T_0": 18, + "lr_scheduler:CosineAnnealingWarmRestarts:T_mult": 1.7405132785152093, + "network_backbone:ResNetBackbone:activation": "relu", + "network_backbone:ResNetBackbone:blocks_per_group_0": 1, + "network_backbone:ResNetBackbone:blocks_per_group_1": 4, + "network_backbone:ResNetBackbone:num_groups": 6, + "network_backbone:ResNetBackbone:num_units_0": 894, + "network_backbone:ResNetBackbone:num_units_1": 395, + "network_backbone:ResNetBackbone:use_dropout": true, + "network_backbone:ResNetBackbone:use_shake_drop": false, + "network_backbone:ResNetBackbone:use_shake_shake": false, + "network_head:fully_connected:num_layers": 4, + "network_init:XavierInit:bias_strategy": "Zero", + "optimizer:RMSpropOptimizer:alpha": 0.6521242194975473, + "optimizer:RMSpropOptimizer:lr": 4.097035283946373e-05, + "optimizer:RMSpropOptimizer:momentum": 0.1792833337110808, + "optimizer:RMSpropOptimizer:weight_decay": 0.006909623450893943, + "scaler:Normalizer:norm": "mean_abs", + "trainer:StandardTrainer:weighted_loss": false, + "network_backbone:ResNetBackbone:blocks_per_group_2": 2, + "network_backbone:ResNetBackbone:blocks_per_group_3": 3, + "network_backbone:ResNetBackbone:blocks_per_group_4": 2, + "network_backbone:ResNetBackbone:blocks_per_group_5": 4, + "network_backbone:ResNetBackbone:blocks_per_group_6": 1, + "network_backbone:ResNetBackbone:dropout_0": 0.6575114752945207, + "network_backbone:ResNetBackbone:dropout_1": 0.28916184819601504, + "network_backbone:ResNetBackbone:dropout_2": 0.09888388652277876, + "network_backbone:ResNetBackbone:dropout_3": 0.791809735686961, + "network_backbone:ResNetBackbone:dropout_4": 0.06432675017963892, + "network_backbone:ResNetBackbone:dropout_5": 0.3015819044494064, + "network_backbone:ResNetBackbone:dropout_6": 0.792332044450592, + "network_backbone:ResNetBackbone:num_units_2": 173, + "network_backbone:ResNetBackbone:num_units_3": 290, + "network_backbone:ResNetBackbone:num_units_4": 633, + "network_backbone:ResNetBackbone:num_units_5": 16, + "network_backbone:ResNetBackbone:num_units_6": 542, + "network_head:fully_connected:activation": "relu", + "network_head:fully_connected:units_layer_1": 429, + "network_head:fully_connected:units_layer_2": 342, + "network_head:fully_connected:units_layer_3": 322 + }, + "13": { + "data_loader:batch_size": 35, + "encoder:__choice__": "OneHotEncoder", + "feature_preprocessor:__choice__": "PowerTransformer", + "imputer:categorical_strategy": "most_frequent", + "imputer:numerical_strategy": "most_frequent", + "lr_scheduler:__choice__": "ExponentialLR", + "network_backbone:__choice__": "ShapedMLPBackbone", + "network_embedding:__choice__": "NoEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "XavierInit", + "optimizer:__choice__": "AdamWOptimizer", + "scaler:__choice__": "Normalizer", + "trainer:__choice__": "MixUpTrainer", + "feature_preprocessor:PowerTransformer:standardize": false, + "lr_scheduler:ExponentialLR:gamma": 0.863670772292724, + "network_backbone:ShapedMLPBackbone:activation": "sigmoid", + "network_backbone:ShapedMLPBackbone:max_units": 957, + "network_backbone:ShapedMLPBackbone:mlp_shape": "long_funnel", + "network_backbone:ShapedMLPBackbone:num_groups": 7, + "network_backbone:ShapedMLPBackbone:output_dim": 16, + "network_backbone:ShapedMLPBackbone:use_dropout": true, + "network_head:fully_connected:num_layers": 3, + "network_init:XavierInit:bias_strategy": "Normal", + "optimizer:AdamWOptimizer:beta1": 0.9298951109018316, + "optimizer:AdamWOptimizer:beta2": 0.9367719861032991, + "optimizer:AdamWOptimizer:lr": 2.3043911799203502e-05, + "optimizer:AdamWOptimizer:weight_decay": 0.08948752020001628, + "scaler:Normalizer:norm": "max", + "trainer:MixUpTrainer:alpha": 0.1848582510096881, + "trainer:MixUpTrainer:weighted_loss": true, + "network_backbone:ShapedMLPBackbone:max_dropout": 0.4933977554884884, + "network_head:fully_connected:activation": "relu", + "network_head:fully_connected:units_layer_1": 105, + "network_head:fully_connected:units_layer_2": 185 + }, + "14": { + "data_loader:batch_size": 154, + "encoder:__choice__": "OneHotEncoder", + "feature_preprocessor:__choice__": "KernelPCA", + "imputer:categorical_strategy": "most_frequent", + "imputer:numerical_strategy": "mean", + "lr_scheduler:__choice__": "StepLR", + "network_backbone:__choice__": "ResNetBackbone", + "network_embedding:__choice__": "LearnedEntityEmbedding", + "network_head:__choice__": "fully_connected", + "network_init:__choice__": "NoInit", + "optimizer:__choice__": "AdamWOptimizer", + "scaler:__choice__": "NoScaler", + "trainer:__choice__": "StandardTrainer", + "feature_preprocessor:KernelPCA:kernel": "sigmoid", + "feature_preprocessor:KernelPCA:n_components": 3, + "lr_scheduler:StepLR:gamma": 0.5658285105415104, + "lr_scheduler:StepLR:step_size": 10, + "network_backbone:ResNetBackbone:activation": "tanh", + "network_backbone:ResNetBackbone:blocks_per_group_0": 2, + "network_backbone:ResNetBackbone:blocks_per_group_1": 2, + "network_backbone:ResNetBackbone:num_groups": 1, + "network_backbone:ResNetBackbone:num_units_0": 623, + "network_backbone:ResNetBackbone:num_units_1": 42, + "network_backbone:ResNetBackbone:use_dropout": false, + "network_backbone:ResNetBackbone:use_shake_drop": true, + "network_backbone:ResNetBackbone:use_shake_shake": true, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_0": 0.7061800992159439, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_1": 0.40404533505032336, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_2": 0.14124612419045746, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_3": 0.24304972767199295, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_4": 0.8403938666630251, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_5": 0.11081539209354929, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_6": 0.5150164644256714, + "network_embedding:LearnedEntityEmbedding:dimension_reduction_7": 0.6185258490472787, + "network_embedding:LearnedEntityEmbedding:min_unique_values_for_embedding": 3, + "network_head:fully_connected:num_layers": 2, + "network_init:NoInit:bias_strategy": "Zero", + "optimizer:AdamWOptimizer:beta1": 0.9639206805787317, + "optimizer:AdamWOptimizer:beta2": 0.9439342949959634, + "optimizer:AdamWOptimizer:lr": 0.05110804312778185, + "optimizer:AdamWOptimizer:weight_decay": 0.026136253949706992, + "trainer:StandardTrainer:weighted_loss": true, + "feature_preprocessor:KernelPCA:coef0": 0.27733876378393374, + "network_backbone:ResNetBackbone:max_shake_drop_probability": 0.4280891218905112, + "network_head:fully_connected:activation": "relu", + "network_head:fully_connected:units_layer_1": 506 + } + } +} \ No newline at end of file diff --git a/test/test_api/test_api.py b/test/test_api/test_api.py index 5f670e59d..5cb271eb0 100644 --- a/test/test_api/test_api.py +++ b/test/test_api/test_api.py @@ -3,7 +3,7 @@ import pathlib import pickle import unittest -from test.test_api.utils import dummy_do_dummy_prediction, dummy_eval_function, dummy_traditional_classification +from test.test_api.utils import dummy_do_dummy_prediction, dummy_eval_function import ConfigSpace as CS from ConfigSpace.configuration_space import Configuration @@ -25,12 +25,10 @@ from autoPyTorch.api.tabular_classification import TabularClassificationTask from autoPyTorch.api.tabular_regression import TabularRegressionTask -from autoPyTorch.data.tabular_validator import TabularInputValidator from autoPyTorch.datasets.resampling_strategy import ( CrossValTypes, HoldoutValTypes, ) -from autoPyTorch.datasets.tabular_dataset import TabularDataset from autoPyTorch.optimizer.smbo import AutoMLSMBO from autoPyTorch.pipeline.base_pipeline import BasePipeline from autoPyTorch.pipeline.components.setup.traditional_ml.traditional_learner import _traditional_learners @@ -575,73 +573,6 @@ def test_portfolio_selection_failure(openml_id, backend, n_samples): ) -@pytest.mark.parametrize('dataset_name', ('iris',)) -@pytest.mark.parametrize('include_traditional', (True, False)) -def test_get_incumbent_results(dataset_name, backend, include_traditional): - # Get the data and check that contents of data-manager make sense - X, y = sklearn.datasets.fetch_openml( - name=dataset_name, - return_X_y=True, as_frame=True - ) - - X_train, X_test, y_train, y_test = sklearn.model_selection.train_test_split( - X, y, random_state=1) - - # Search for a good configuration - estimator = TabularClassificationTask( - backend=backend, - resampling_strategy=HoldoutValTypes.holdout_validation, - ) - - InputValidator = TabularInputValidator( - is_classification=True, - ) - - # Fit a input validator to check the provided data - # Also, an encoder is fit to both train and test data, - # to prevent unseen categories during inference - InputValidator.fit(X_train=X_train, y_train=y_train, X_test=X_test, y_test=y_test) - - dataset = TabularDataset( - X=X_train, Y=y_train, - X_test=X_test, Y_test=y_test, - validator=InputValidator, - resampling_strategy=estimator.resampling_strategy, - resampling_strategy_args=estimator.resampling_strategy_args, - ) - - pipeline_run_history = RunHistory() - pipeline_run_history.load_json(os.path.join(os.path.dirname(__file__), '.tmp_api/runhistory.json'), - estimator.get_search_space(dataset)) - - estimator._do_dummy_prediction = unittest.mock.MagicMock() - - with unittest.mock.patch.object(AutoMLSMBO, 'run_smbo') as AutoMLSMBOMock: - with unittest.mock.patch.object(TabularClassificationTask, '_do_traditional_prediction', - new=dummy_traditional_classification): - AutoMLSMBOMock.return_value = (pipeline_run_history, {}, 'epochs') - estimator.search( - X_train=X_train, y_train=y_train, - X_test=X_test, y_test=y_test, - optimize_metric='accuracy', - total_walltime_limit=150, - func_eval_time_limit_secs=50, - enable_traditional_pipeline=True, - load_models=False, - ) - config, results = estimator.get_incumbent_results(include_traditional=include_traditional) - assert isinstance(config, Configuration) - assert isinstance(results, dict) - - run_history_data = estimator.run_history.data - costs = [run_value.cost for run_key, run_value in run_history_data.items() if run_value.additional_info is not None - and (run_value.additional_info['configuration_origin'] != 'traditional' or include_traditional)] - assert results['opt_loss']['accuracy'] == min(costs) - - if not include_traditional: - assert results['configuration_origin'] != 'traditional' - - # TODO: Make faster when https://github.com/automl/Auto-PyTorch/pull/223 is incorporated @pytest.mark.parametrize("fit_dictionary_tabular", ['classification_categorical_only'], indirect=True) def test_do_traditional_pipeline(fit_dictionary_tabular): diff --git a/test/test_api/test_results_manager.py b/test/test_api/test_results_manager.py new file mode 100644 index 000000000..4c6e7a7ae --- /dev/null +++ b/test/test_api/test_results_manager.py @@ -0,0 +1,232 @@ +import json +import os +from test.test_api.utils import make_dict_run_history_data +from unittest.mock import MagicMock + +import ConfigSpace.hyperparameters as CSH +from ConfigSpace.configuration_space import Configuration, ConfigurationSpace + +import numpy as np + +import pytest + +from smac.runhistory.runhistory import RunHistory, StatusType + +from autoPyTorch.api.base_task import BaseTask +from autoPyTorch.api.results_manager import ResultsManager, STATUS2MSG, SearchResults, cost2metric +from autoPyTorch.metrics import accuracy, balanced_accuracy, log_loss + + +def _check_status(status): + """ Based on runhistory_B.json """ + ans = [ + STATUS2MSG[StatusType.SUCCESS], STATUS2MSG[StatusType.SUCCESS], + STATUS2MSG[StatusType.SUCCESS], STATUS2MSG[StatusType.SUCCESS], + STATUS2MSG[StatusType.SUCCESS], STATUS2MSG[StatusType.SUCCESS], + STATUS2MSG[StatusType.CRASHED], STATUS2MSG[StatusType.SUCCESS], + STATUS2MSG[StatusType.SUCCESS], STATUS2MSG[StatusType.SUCCESS], + STATUS2MSG[StatusType.SUCCESS], STATUS2MSG[StatusType.SUCCESS], + STATUS2MSG[StatusType.SUCCESS], STATUS2MSG[StatusType.SUCCESS], + STATUS2MSG[StatusType.TIMEOUT], STATUS2MSG[StatusType.TIMEOUT], + ] + assert isinstance(status, list) + assert isinstance(status[0], str) + assert status == ans + + +def _check_costs(costs): + """ Based on runhistory_B.json """ + ans = [0.15204678362573099, 0.4444444444444444, 0.5555555555555556, 0.29824561403508776, + 0.4444444444444444, 0.4444444444444444, 1.0, 0.5555555555555556, 0.4444444444444444, + 0.15204678362573099, 0.15204678362573099, 0.4035087719298246, 0.4444444444444444, + 0.4444444444444444, 1.0, 1.0] + assert np.allclose(1 - np.array(costs), ans) + assert isinstance(costs, np.ndarray) + assert costs.dtype is np.dtype(np.float) + + +def _check_fit_times(fit_times): + """ Based on runhistory_B.json """ + ans = [3.154788017272949, 3.2763524055480957, 22.723600149154663, 4.990685224533081, 10.684926509857178, + 9.947429180145264, 11.687273979187012, 8.478890419006348, 5.485020637512207, 11.514830589294434, + 15.370736837387085, 23.846530199050903, 6.757539510726929, 15.061991930007935, 50.010520696640015, + 22.011935234069824] + + assert np.allclose(fit_times, ans) + assert isinstance(fit_times, np.ndarray) + assert fit_times.dtype is np.dtype(np.float) + + +def _check_budgets(budgets): + """ Based on runhistory_B.json """ + ans = [5.555555555555555, 5.555555555555555, 5.555555555555555, 5.555555555555555, + 5.555555555555555, 5.555555555555555, 5.555555555555555, 5.555555555555555, + 5.555555555555555, 16.666666666666664, 50.0, 16.666666666666664, 16.666666666666664, + 16.666666666666664, 50.0, 50.0] + assert np.allclose(budgets, ans) + assert isinstance(budgets, list) + assert isinstance(budgets[0], float) + + +def _check_additional_infos(status_types, additional_infos): + for i, status in enumerate(status_types): + info = additional_infos[i] + if status in (STATUS2MSG[StatusType.SUCCESS], STATUS2MSG[StatusType.DONOTADVANCE]): + metric_info = info.get('opt_loss', None) + assert metric_info is not None + elif info is not None: + metric_info = info.get('opt_loss', None) + assert metric_info is None + + +def _check_metric_dict(metric_dict, status_types): + assert isinstance(metric_dict['accuracy'], list) + assert metric_dict['accuracy'][0] > 0 + assert isinstance(metric_dict['balanced_accuracy'], list) + assert metric_dict['balanced_accuracy'][0] > 0 + + for key, vals in metric_dict.items(): + # ^ is a XOR operator + # True and False / False and True must be fulfilled + assert all([(s == STATUS2MSG[StatusType.SUCCESS]) ^ isnan + for s, isnan in zip(status_types, np.isnan(vals))]) + + +def test_extract_results_from_run_history(): + # test the raise error for the `status_msg is None` + run_history = RunHistory() + cs = ConfigurationSpace() + config = Configuration(cs, {}) + run_history.add( + config=config, + cost=0.0, + time=1.0, + status=StatusType.CAPPED, + ) + with pytest.raises(ValueError) as excinfo: + SearchResults(metric=accuracy, scoring_functions=[], run_history=run_history) + + assert excinfo._excinfo[0] == ValueError + + +def test_search_results_sprint_statistics(): + api = BaseTask() + for method in ['get_search_results', 'sprint_statistics', 'get_incumbent_results']: + with pytest.raises(RuntimeError) as excinfo: + getattr(api, method)() + + assert excinfo._excinfo[0] == RuntimeError + + run_history_data = json.load(open(os.path.join(os.path.dirname(__file__), + '.tmp_api/runhistory_B.json'), + mode='r'))['data'] + api._results_manager.run_history = MagicMock() + api.run_history.empty = MagicMock(return_value=False) + + # The run_history has 16 runs + 1 run interruption ==> 16 runs + api.run_history.data = make_dict_run_history_data(run_history_data) + api._metric = accuracy + api.dataset_name = 'iris' + api._scoring_functions = [accuracy, balanced_accuracy] + api.search_space = MagicMock(spec=ConfigurationSpace) + search_results = api.get_search_results() + + _check_status(search_results.status_types) + _check_costs(search_results.opt_scores) + _check_fit_times(search_results.fit_times) + _check_budgets(search_results.budgets) + _check_metric_dict(search_results.metric_dict, search_results.status_types) + _check_additional_infos(status_types=search_results.status_types, + additional_infos=search_results.additional_infos) + + # config_ids can duplicate because of various budget size + config_ids = [1, 2, 3, 4, 5, 6, 7, 8, 9, 1, 1, 10, 11, 12, 10, 13] + assert config_ids == search_results.config_ids + + # assert that contents of search_results are of expected types + assert isinstance(search_results.rank_test_scores, np.ndarray) + assert search_results.rank_test_scores.dtype is np.dtype(np.int) + assert isinstance(search_results.configs, list) + + n_success, n_timeout, n_memoryout, n_crashed = 13, 2, 0, 1 + msg = ["autoPyTorch results:", f"\tDataset name: {api.dataset_name}", + f"\tOptimisation Metric: {api._metric.name}", + f"\tBest validation score: {max(search_results.opt_scores)}", + "\tNumber of target algorithm runs: 16", f"\tNumber of successful target algorithm runs: {n_success}", + f"\tNumber of crashed target algorithm runs: {n_crashed}", + f"\tNumber of target algorithms that exceeded the time limit: {n_timeout}", + f"\tNumber of target algorithms that exceeded the memory limit: {n_memoryout}"] + + assert isinstance(api.sprint_statistics(), str) + assert all([m1 == m2 for m1, m2 in zip(api.sprint_statistics().split("\n"), msg)]) + + +@pytest.mark.parametrize('run_history', (None, RunHistory())) +def test_check_run_history(run_history): + manager = ResultsManager() + manager.run_history = run_history + + with pytest.raises(RuntimeError) as excinfo: + manager._check_run_history() + + assert excinfo._excinfo[0] == RuntimeError + + +T, NT = 'traditional', 'non-traditional' +SCORES = [0.1 * (i + 1) for i in range(10)] + + +@pytest.mark.parametrize('include_traditional', (True, False)) +@pytest.mark.parametrize('metric', (accuracy, log_loss)) +@pytest.mark.parametrize('origins', ([T] * 5 + [NT] * 5, [T, NT] * 5, [NT] * 5 + [T] * 5)) +@pytest.mark.parametrize('scores', (SCORES, SCORES[::-1])) +def test_get_incumbent_results(include_traditional, metric, origins, scores): + manager = ResultsManager() + cs = ConfigurationSpace() + cs.add_hyperparameter(CSH.UniformFloatHyperparameter('a', lower=0, upper=1)) + + configs = [0.1 * (i + 1) for i in range(len(scores))] + if metric.name == "log_loss": + # This is to detect mis-computation in reversion + metric._optimum = 0.1 + + best_cost, best_idx = np.inf, -1 + for idx, (a, origin, score) in enumerate(zip(configs, origins, scores)): + config = Configuration(cs, {'a': a}) + + # conversion defined in: + # autoPyTorch/pipeline/components/training/metrics/utils.py::calculate_loss + cost = metric._optimum - metric._sign * score + manager.run_history.add( + config=config, + cost=cost, + time=1.0, + status=StatusType.SUCCESS, + additional_info={'opt_loss': {metric.name: score}, + 'configuration_origin': origin} + ) + if cost > best_cost: + continue + + if include_traditional: + best_cost, best_idx = cost, idx + elif origin != T: + best_cost, best_idx = cost, idx + + incumbent_config, incumbent_results = manager.get_incumbent_results( + metric=metric, + include_traditional=include_traditional + ) + + assert isinstance(incumbent_config, Configuration) + assert isinstance(incumbent_results, dict) + best_score, best_a = scores[best_idx], configs[best_idx] + assert np.allclose( + [best_score, best_score, best_a], + [cost2metric(best_cost, metric), + incumbent_results['opt_loss'][metric.name], + incumbent_config['a']] + ) + + if not include_traditional: + assert incumbent_results['configuration_origin'] != T diff --git a/test/test_api/utils.py b/test/test_api/utils.py index d09f66c9a..a8c258fe9 100644 --- a/test/test_api/utils.py +++ b/test/test_api/utils.py @@ -1,6 +1,6 @@ import os -from smac.runhistory.runhistory import DataOrigin, RunHistory +from smac.runhistory.runhistory import DataOrigin, RunHistory, RunKey, RunValue, StatusType from autoPyTorch.constants import REGRESSION_TASKS from autoPyTorch.evaluation.abstract_evaluator import ( @@ -113,3 +113,24 @@ def dummy_eval_function( def dummy_do_dummy_prediction(): return + + +def make_dict_run_history_data(data): + run_history_data = dict() + for row in data: + run_key = RunKey( + config_id=row[0][0], + instance_id=row[0][1], + seed=row[0][2], + budget=row[0][3]) + + run_value = RunValue( + cost=row[1][0], + time=row[1][1], + status=getattr(StatusType, row[1][2]['__enum__'].split(".")[-1]), + starttime=row[1][3], + endtime=row[1][4], + additional_info=row[1][5], + ) + run_history_data[run_key] = run_value + return run_history_data