From 925295514e124b8b23e1d34c5c197f4ecf92a367 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Thu, 11 Jan 2024 18:40:26 +0000 Subject: [PATCH] Rename and improve tunable vs config API and documentation --- mlos_bench/mlos_bench/run.py | 2 +- .../mlos_bench/storage/base_experiment_data.py | 2 +- mlos_bench/mlos_bench/storage/base_storage.py | 11 +++++++++-- mlos_bench/mlos_bench/storage/base_trial_data.py | 14 +++++++++----- mlos_bench/mlos_bench/storage/sql/experiment.py | 6 +++++- mlos_bench/mlos_bench/storage/sql/trial_data.py | 10 +++++++--- 6 files changed, 32 insertions(+), 13 deletions(-) diff --git a/mlos_bench/mlos_bench/run.py b/mlos_bench/mlos_bench/run.py index 70a416e4b0..7240097ba5 100755 --- a/mlos_bench/mlos_bench/run.py +++ b/mlos_bench/mlos_bench/run.py @@ -109,7 +109,7 @@ def _optimize(*, tunables = opt_context.suggest() if config_id > 0: - tunable_values = exp.load_config(config_id) + tunable_values = exp.load_tunable_config(config_id) tunables.assign(tunable_values) _LOG.info("Load config from storage: %d", config_id) if _LOG.isEnabledFor(logging.DEBUG): diff --git a/mlos_bench/mlos_bench/storage/base_experiment_data.py b/mlos_bench/mlos_bench/storage/base_experiment_data.py index 22025f290a..f9240d8fe7 100644 --- a/mlos_bench/mlos_bench/storage/base_experiment_data.py +++ b/mlos_bench/mlos_bench/storage/base_experiment_data.py @@ -79,6 +79,6 @@ def results(self) -> pandas.DataFrame: results : pandas.DataFrame A DataFrame with configurations and results from all trials of the experiment. Has columns [trial_id, config_id, ts_start, ts_end, status] - followed by config parameters and trial results. The latter can be NULLs + followed by tunable config parameters and trial results. The latter can be NULLs if the trial was not successful. """ diff --git a/mlos_bench/mlos_bench/storage/base_storage.py b/mlos_bench/mlos_bench/storage/base_storage.py index e5000b5d6f..ef1ad32da0 100644 --- a/mlos_bench/mlos_bench/storage/base_storage.py +++ b/mlos_bench/mlos_bench/storage/base_storage.py @@ -185,7 +185,7 @@ def merge(self, experiment_ids: List[str]) -> None: """ @abstractmethod - def load_config(self, config_id: int) -> Dict[str, Any]: + def load_tunable_config(self, config_id: int) -> Dict[str, Any]: """ Load tunable values for a given config ID. """ @@ -277,7 +277,9 @@ def config_id(self) -> int: @property def tunables(self) -> TunableGroups: """ - Tunable parameters of the current trial. + Tunable parameters of the current trial + + (e.g., application Environment's "config") """ return self._tunables @@ -285,6 +287,11 @@ def config(self, global_config: Optional[Dict[str, Any]] = None) -> Dict[str, An """ Produce a copy of the global configuration updated with the parameters of the current trial. + + Note: this is not the target Environment's "config" (i.e., tunable + params), but rather the internal "config" which consists of a + combination of somewhat more static variables defined in the json config + files. """ config = self._config.copy() config.update(global_config or {}) diff --git a/mlos_bench/mlos_bench/storage/base_trial_data.py b/mlos_bench/mlos_bench/storage/base_trial_data.py index 58d0c9bb43..71ad7e75a4 100644 --- a/mlos_bench/mlos_bench/storage/base_trial_data.py +++ b/mlos_bench/mlos_bench/storage/base_trial_data.py @@ -80,14 +80,16 @@ def status(self) -> Status: @property @abstractmethod - def config(self) -> pandas.DataFrame: + def tunable_config(self) -> pandas.DataFrame: """ - Retrieve the trials' configuration from the storage. + Retrieve the trials' tunable configuration from the storage. + + Note: this corresponds to the Trial object's "tunables" property. Returns ------- config : pandas.DataFrame - A dataframe with the configuration of the current trial. + A dataframe with the tunable configuration of the current trial. It has two `str` columns, "parameter" and "value". """ @@ -124,11 +126,13 @@ def telemetry(self) -> pandas.DataFrame: @abstractmethod def metadata(self) -> pandas.DataFrame: """ - Retrieve the trials' metadata. + Retrieve the trials' metadata parameters. + + Note: this corresponds to the Trial object's "config" property. Returns ------- - config : pandas.DataFrame + metadata : pandas.DataFrame An optional dataframe with the metadata associated with the trial. It has two `str` columns, "parameter" and "value". Returns an empty dataframe if there is no metadata. diff --git a/mlos_bench/mlos_bench/storage/sql/experiment.py b/mlos_bench/mlos_bench/storage/sql/experiment.py index 8a271a9556..64160becef 100644 --- a/mlos_bench/mlos_bench/storage/sql/experiment.py +++ b/mlos_bench/mlos_bench/storage/sql/experiment.py @@ -89,7 +89,7 @@ def merge(self, experiment_ids: List[str]) -> None: _LOG.info("Merge: %s <- %s", self._experiment_id, experiment_ids) raise NotImplementedError() - def load_config(self, config_id: int) -> Dict[str, Any]: + def load_tunable_config(self, config_id: int) -> Dict[str, Any]: with self._engine.connect() as conn: return self._get_params(conn, self._schema.config_param, config_id=config_id) @@ -216,10 +216,14 @@ def new_trial(self, tunables: TunableGroups, ts_start=datetime.utcnow(), status='PENDING', )) + + # Note: config here is the framework config, not the target + # environment config (i.e., tunables). if config is not None: self._save_params( conn, self._schema.trial_param, config, exp_id=self._experiment_id, trial_id=self._trial_id) + trial = Trial( engine=self._engine, schema=self._schema, diff --git a/mlos_bench/mlos_bench/storage/sql/trial_data.py b/mlos_bench/mlos_bench/storage/sql/trial_data.py index 75d6d05e21..8a0dd187b4 100644 --- a/mlos_bench/mlos_bench/storage/sql/trial_data.py +++ b/mlos_bench/mlos_bench/storage/sql/trial_data.py @@ -42,9 +42,11 @@ def __init__(self, *, self._schema = schema @property - def config(self) -> pandas.DataFrame: + def tunable_config(self) -> pandas.DataFrame: """ - Retrieve the trials' configuration from the storage. + Retrieve the trials' tunable configuration from the storage. + + Note: this corresponds to the Trial object's "tunables" property. """ with self._engine.connect() as conn: cur_config = conn.execute( @@ -98,7 +100,9 @@ def telemetry(self) -> pandas.DataFrame: @property def metadata(self) -> pandas.DataFrame: """ - Retrieve the trials' metadata. + Retrieve the trials' metadata params. + + Note: this corresponds to the Trial object's "config" property. """ with self._engine.connect() as conn: cur_params = conn.execute(