Make sure One Shot Optimizer always returns the same suggestion

mlos_bench/mlos_bench/DEVNOTES.md

@@ -34,7 +34,7 @@ Each `Environment` config is a JSON5 file with the following structure:
  }
  // Environment constructor parameters
  // (specific to the Environment class being instantiated):
- "seed": 42,
+ "mock_env_seed": 42,
  // ...
  }
 }

mlos_bench/mlos_bench/config/environments/mock/mock_env.jsonc

@@ -19,8 +19,8 @@
  "linux-hugepages-2048kB",
  "redis"
  ],
- "seed": 42, // Seed for the random noise generator. Omit to produce noise-free data.
- "range": [60, 120], // Range of the generated output values of the benchmark.
- "metrics": ["score"] // Names of fake benchmark metrics to generate. (Default is one metric, "score").
+ "mock_env_seed": 42, // Seed for the random noise generator. Omit or set to 0 to produce noise-free data.
+ "mock_env_range": [60, 120], // Range of the generated output values of the benchmark.
+ "mock_env_metrics": ["score"] // Names of fake benchmark metrics to generate. (Default is one metric, "score").
  }
 }

mlos_bench/mlos_bench/config/schemas/environments/mock-env-subschema.json

@@ -19,12 +19,12 @@
  },
  {
  "properties": {
- "seed": {
+ "mock_env_seed": {
  "type": "integer",
  "description": "Seed for the random number generator",
  "default": 0
  },
- "range": {
+ "mock_env_range": {
  "type": "array",
  "description": "Range of the random number generator",
  "items": {
@@ -33,7 +33,7 @@
  "minItems": 2,
  "maxItems": 2
  },
- "metrics": {
+ "mock_env_metrics": {
  "type": "array",
  "description": "Names of fake benchmark metrics to be generate",
  "items": {

mlos_bench/mlos_bench/environments/mock_env.py

@@ -17,7 +17,6 @@
 from mlos_bench.environments.status import Status
 from mlos_bench.environments.base_environment import Environment
 from mlos_bench.tunables import Tunable, TunableGroups, TunableValue
-from mlos_bench.util import nullable
 
 _LOG = logging.getLogger(__name__)
 
@@ -49,18 +48,18 @@ def __init__(self,
  global_config : dict
  Free-format dictionary of global parameters (e.g., security credentials)
  to be mixed in into the "const_args" section of the local config.
- Optional arguments are `seed`, `range`, and `metrics`.
+ Optional arguments are `mock_env_seed`, `mock_env_range`, and `mock_env_metrics`.
  tunables : TunableGroups
  A collection of tunable parameters for *all* environments.
  service: Service
  An optional service object. Not used by this class.
  """
  super().__init__(name=name, config=config, global_config=global_config,
  tunables=tunables, service=service)
- seed = self.config.get("seed")
- self._random = nullable(random.Random, seed)
- self._range = self.config.get("range")
- self._metrics = self.config.get("metrics", ["score"])
+ seed = int(self.config.get("mock_env_seed", 0))
+ self._random = random.Random(seed) if seed > 0 else None
+ self._range = self.config.get("mock_env_range")
+ self._metrics = self.config.get("mock_env_metrics", ["score"])
  self._is_ready = True
 
  def run(self) -> Tuple[Status, datetime, Optional[Dict[str, TunableValue]]]:

mlos_bench/mlos_bench/optimizers/mock_optimizer.py

@@ -61,7 +61,7 @@ def suggest(self) -> TunableGroups:
  """
  tunables = super().suggest()
  if self._start_with_defaults:
- _LOG.info("Use default values for the first trial")
+ _LOG.info("Use default tunable values")
  self._start_with_defaults = False
  else:
  for (tunable, _group) in tunables:

mlos_bench/mlos_bench/optimizers/one_shot_optimizer.py

@@ -18,7 +18,7 @@
 
 class OneShotOptimizer(MockOptimizer):
  """
- Mock optimizer that proposes a single configuration and returns.
+ No-op optimizer that proposes a single configuration and returns.
  Explicit configs (partial or full) are possible using configuration files.
  """
 
@@ -33,6 +33,14 @@ def __init__(self,
  _LOG.info("Run a single iteration for: %s", self._tunables)
  self._max_iter = 1 # Always run for just one iteration.
 
+ def suggest(self) -> TunableGroups:
+ """
+ Always produce the same (initial) suggestion.
+ """
+ tunables = super().suggest()
+ self._start_with_defaults = True
+ return tunables
+
  @property
  def supports_preload(self) -> bool:
  return False

mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-bad-metric-type.jsonc

@@ -2,7 +2,7 @@
  "name": "mock_env-full",
  "class": "mlos_bench.environments.mock_env.MockEnv",
  "config": {
- "metrics": [
+ "mock_env_metrics": [
  {"bad": "metric type"}
  ]
  }

mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-const-args-list-obj.jsonc

@@ -2,7 +2,7 @@
  "name": "partial mock env",
  "class": "mlos_bench.environments.MockEnv",
  "config": {
- "seed": 42,
+ "mock_env_seed": 42,
  "const_args": {
  "foo": "bar",
  "int": 1,

mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-const-args-nested-list.jsonc

@@ -2,7 +2,7 @@
  "name": "partial mock env",
  "class": "mlos_bench.environments.MockEnv",
  "config": {
- "seed": 42,
+ "mock_env_seed": 42,
  "const_args": {
  "foo": "bar",
  "int": 1,

mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-missing-metric.jsonc

@@ -2,7 +2,7 @@
  "name": "mock_env-full",
  "class": "mlos_bench.environments.mock_env.MockEnv",
  "config": {
- "metrics": [
+ "mock_env_metrics": [
  // needs at least one element
  ]
  }

mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-missing-name.jsonc

@@ -2,6 +2,6 @@
  // "name": "missing name is invalid",
  "class": "mlos_bench.environments.MockEnv",
  "config": {
- "seed": 42
+ "mock_env_seed": 42
  }
 }

mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/good/full/mock_env-full.jsonc

@@ -20,9 +20,9 @@
  "required_args": [
  "foo"
  ],
- "range": [0, 1],
- "seed": 42,
- "metrics": [
+ "mock_env_range": [0, 1],
+ "mock_env_seed": 42,
+ "mock_env_metrics": [
  "latency",
  "cost"
  ]

mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/good/partial/mock_env-const-args.jsonc

@@ -2,7 +2,7 @@
  "name": "partial mock env",
  "class": "mlos_bench.environments.MockEnv",
  "config": {
- "seed": 42,
+ "mock_env_seed": 42,
  "const_args": {
  "foo": "bar",
  "int": 1,

mlos_bench/mlos_bench/tests/conftest.py

@@ -40,9 +40,9 @@ def mock_env(tunable_groups: TunableGroups) -> MockEnv:
  name="Test Env",
  config={
  "tunable_params": ["provision", "boot", "kernel"],
- "seed": SEED,
- "range": [60, 120],
- "metrics": ["score"],
+ "mock_env_seed": SEED,
+ "mock_env_range": [60, 120],
+ "mock_env_metrics": ["score"],
  },
  tunables=tunable_groups
  )
@@ -57,8 +57,9 @@ def mock_env_no_noise(tunable_groups: TunableGroups) -> MockEnv:
  name="Test Env No Noise",
  config={
  "tunable_params": ["provision", "boot", "kernel"],
- "range": [60, 120],
- "metrics": ["score", "other_score"],
+ "mock_env_seed": 0,
+ "mock_env_range": [60, 120],
+ "mock_env_metrics": ["score", "other_score"],
  },
  tunables=tunable_groups
  )

mlos_bench/mlos_bench/tests/environments/composite_env_test.py

@@ -41,8 +41,8 @@ def composite_env(tunable_groups: TunableGroups) -> CompositeEnv:
  "EnvId": 1,
  },
  "required_args": ["vmName", "someConst", "global_param"],
- "range": [60, 120],
- "metrics": ["score"],
+ "mock_env_range": [60, 120],
+ "mock_env_metrics": ["score"],
  }
  },
  {
@@ -56,8 +56,8 @@ def composite_env(tunable_groups: TunableGroups) -> CompositeEnv:
  "global_param": "local"
  },
  "required_args": ["vmName"],
- "range": [60, 120],
- "metrics": ["score"],
+ "mock_env_range": [60, 120],
+ "mock_env_metrics": ["score"],
  }
  },
  {
@@ -70,8 +70,8 @@ def composite_env(tunable_groups: TunableGroups) -> CompositeEnv:
  "EnvId": 3,
  },
  "required_args": ["vmName", "vm_server_name", "vm_client_name"],
- "range": [60, 120],
- "metrics": ["score"],
+ "mock_env_range": [60, 120],
+ "mock_env_metrics": ["score"],
  }
  }
  ]
@@ -193,8 +193,8 @@ def nested_composite_env(tunable_groups: TunableGroups) -> CompositeEnv:
  "vm_server_name",
  "global_param"
  ],
- "range": [60, 120],
- "metrics": ["score"],
+ "mock_env_range": [60, 120],
+ "mock_env_metrics": ["score"],
  }
  },
  # ...
@@ -218,8 +218,8 @@ def nested_composite_env(tunable_groups: TunableGroups) -> CompositeEnv:
  "config": {
  "tunable_params": ["boot"],
  "required_args": ["vmName", "EnvId", "vm_client_name"],
- "range": [60, 120],
- "metrics": ["score"],
+ "mock_env_range": [60, 120],
+ "mock_env_metrics": ["score"],
  }
  },
  # ...

mlos_bench/mlos_bench/tests/launcher_in_process_test.py

@@ -17,11 +17,14 @@
  ("argv", "expected_score"), [
  ([
  "--config", "mlos_bench/mlos_bench/tests/config/cli/mock-bench.jsonc",
- ], 65.6742),
+ "--trial_config_repeat_count", "5",
+ "--mock_env_seed", "0", # Deterministic Mock Environment.
+ ], 67.40329),
  ([
  "--config", "mlos_bench/mlos_bench/tests/config/cli/mock-opt.jsonc",
  "--trial_config_repeat_count", "3",
  "--max_suggestions", "3",
+ "--mock_env_seed", "42", # Noisy Mock Environment.
  ], 64.53897),
  ]
 )

mlos_bench/mlos_bench/tests/launcher_run_test.py

@@ -78,10 +78,12 @@ def test_launch_main_app_bench(root_path: str, local_exec_service: LocalExecServ
  """
  _launch_main_app(
  root_path, local_exec_service,
- "--config mlos_bench/mlos_bench/tests/config/cli/mock-bench.jsonc",
+ "--config mlos_bench/mlos_bench/tests/config/cli/mock-bench.jsonc" +
+ " --trial_config_repeat_count 5" +
+ " --mock_env_seed 0", # Deterministic Mock Environment.
  [
  f"^{_RE_DATE} run\\.py:\\d+ " +
- r"_main INFO Final score: \{'score': 65\.67\d+\}\s*$",
+ r"_main INFO Final score: \{'score': 67\.40\d+\}\s*$",
  ]
  )
 
@@ -93,7 +95,10 @@ def test_launch_main_app_opt(root_path: str, local_exec_service: LocalExecServic
  """
  _launch_main_app(
  root_path, local_exec_service,
- "--config mlos_bench/mlos_bench/tests/config/cli/mock-opt.jsonc --trial_config_repeat_count 3 --max_suggestions 3",
+ "--config mlos_bench/mlos_bench/tests/config/cli/mock-opt.jsonc" +
+ " --trial_config_repeat_count 3" +
+ " --max_suggestions 3" +
+ " --mock_env_seed 42", # Noisy Mock Environment.
  [
  # Iteration 1: Expect first value to be the baseline
  f"^{_RE_DATE} mlos_core_optimizer\\.py:\\d+ " +