Better naming and values for MockEnv parameters + unit tests

mlos_bench/mlos_bench/DEVNOTES.md

@@ -34,7 +34,7 @@ Each `Environment` config is a JSON5 file with the following structure:
  }
  // Environment constructor parameters
  // (specific to the Environment class being instantiated):
- "seed": 42,
+ "mock_env_seed": 42,
  // ...
  }
 }

mlos_bench/mlos_bench/config/environments/mock/mock_env.jsonc

@@ -19,8 +19,8 @@
  "linux-hugepages-2048kB",
  "redis"
  ],
- "seed": 42, // Seed for the random noise generator. Omit to produce noise-free data.
- "range": [60, 120], // Range of the generated output values of the benchmark.
- "metrics": ["score"] // Names of fake benchmark metrics to generate. (Default is one metric, "score").
+ "mock_env_seed": 42, // Seed for the random noise generator. Omit or set to 0 to produce noise-free data.
+ "mock_env_range": [60, 120], // Range of the generated output values of the benchmark.
+ "mock_env_metrics": ["score"] // Names of fake benchmark metrics to generate. (Default is one metric, "score").
  }
 }

mlos_bench/mlos_bench/config/schemas/environments/mock-env-subschema.json

@@ -19,12 +19,12 @@
  },
  {
  "properties": {
- "seed": {
+ "mock_env_seed": {
  "type": "integer",
- "description": "Seed for the random number generator",
- "default": 0
+ "description": "Seed for the random number generator. Set to -1 for deterministic behavior, 0 for default randomness.",
+ "default": -1
  },
- "range": {
+ "mock_env_range": {
  "type": "array",
  "description": "Range of the random number generator",
  "items": {
@@ -33,7 +33,7 @@
  "minItems": 2,
  "maxItems": 2
  },
- "metrics": {
+ "mock_env_metrics": {
  "type": "array",
  "description": "Names of fake benchmark metrics to be generate",
  "items": {

mlos_bench/mlos_bench/environments/mock_env.py

@@ -17,7 +17,6 @@
 from mlos_bench.environments.status import Status
 from mlos_bench.environments.base_environment import Environment
 from mlos_bench.tunables import Tunable, TunableGroups, TunableValue
-from mlos_bench.util import nullable
 
 _LOG = logging.getLogger(__name__)
 
@@ -49,18 +48,19 @@ def __init__(self,
  global_config : dict
  Free-format dictionary of global parameters (e.g., security credentials)
  to be mixed in into the "const_args" section of the local config.
- Optional arguments are `seed`, `range`, and `metrics`.
+ Optional arguments are `mock_env_seed`, `mock_env_range`, and `mock_env_metrics`.
+ Set `mock_env_seed` to -1 for deterministic behavior, 0 for default randomness.
  tunables : TunableGroups
  A collection of tunable parameters for *all* environments.
  service: Service
  An optional service object. Not used by this class.
  """
  super().__init__(name=name, config=config, global_config=global_config,
  tunables=tunables, service=service)
- seed = self.config.get("seed")
- self._random = nullable(random.Random, seed)
- self._range = self.config.get("range")
- self._metrics = self.config.get("metrics", ["score"])
+ seed = int(self.config.get("mock_env_seed", -1))
+ self._random = random.Random(seed or None) if seed >= 0 else None
+ self._range = self.config.get("mock_env_range")
+ self._metrics = self.config.get("mock_env_metrics", ["score"])
  self._is_ready = True
 
  def run(self) -> Tuple[Status, datetime, Optional[Dict[str, TunableValue]]]:

mlos_bench/mlos_bench/optimizers/mock_optimizer.py

@@ -61,7 +61,7 @@ def suggest(self) -> TunableGroups:
  """
  tunables = super().suggest()
  if self._start_with_defaults:
- _LOG.info("Use default values for the first trial")
+ _LOG.info("Use default tunable values")
  self._start_with_defaults = False
  else:
  for (tunable, _group) in tunables:

mlos_bench/mlos_bench/tests/config/cli/mock-bench.jsonc

@@ -14,9 +14,10 @@
 
  "environment": "environments/mock/mock_env.jsonc",
 
- "tunable_values": [
- "tunable-values/tunable-values-example.jsonc"
- ],
+ // Use default values for tunables unless otherwise specified
+ // "tunable_values": [
+ // "tunable-values/tunable-values-example.jsonc"
+ // ],
 
  "globals": ["globals/global_test_config.jsonc"],
 

mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-bad-metric-type.jsonc

@@ -2,7 +2,7 @@
  "name": "mock_env-full",
  "class": "mlos_bench.environments.mock_env.MockEnv",
  "config": {
- "metrics": [
+ "mock_env_metrics": [
  {"bad": "metric type"}
  ]
  }

mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-const-args-list-obj.jsonc

@@ -2,7 +2,7 @@
  "name": "partial mock env",
  "class": "mlos_bench.environments.MockEnv",
  "config": {
- "seed": 42,
+ "mock_env_seed": 42,
  "const_args": {
  "foo": "bar",
  "int": 1,

mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-const-args-nested-list.jsonc

@@ -2,7 +2,7 @@
  "name": "partial mock env",
  "class": "mlos_bench.environments.MockEnv",
  "config": {
- "seed": 42,
+ "mock_env_seed": 42,
  "const_args": {
  "foo": "bar",
  "int": 1,

mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-missing-metric.jsonc

@@ -2,7 +2,7 @@
  "name": "mock_env-full",
  "class": "mlos_bench.environments.mock_env.MockEnv",
  "config": {
- "metrics": [
+ "mock_env_metrics": [
  // needs at least one element
  ]
  }

mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/bad/invalid/mock_env-missing-name.jsonc

@@ -2,6 +2,6 @@
  // "name": "missing name is invalid",
  "class": "mlos_bench.environments.MockEnv",
  "config": {
- "seed": 42
+ "mock_env_seed": 42
  }
 }

mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/good/full/mock_env-full.jsonc

@@ -20,9 +20,9 @@
  "required_args": [
  "foo"
  ],
- "range": [0, 1],
- "seed": 42,
- "metrics": [
+ "mock_env_range": [0, 1],
+ "mock_env_seed": 42,
+ "mock_env_metrics": [
  "latency",
  "cost"
  ]

mlos_bench/mlos_bench/tests/config/schemas/environments/test-cases/good/partial/mock_env-const-args.jsonc

@@ -2,7 +2,7 @@
  "name": "partial mock env",
  "class": "mlos_bench.environments.MockEnv",
  "config": {
- "seed": 42,
+ "mock_env_seed": 42,
  "const_args": {
  "foo": "bar",
  "int": 1,

mlos_bench/mlos_bench/tests/config/tunable-values/tunable-values-example.jsonc

@@ -2,8 +2,10 @@
 {
  "$schema": "https://raw.githubusercontent.com/microsoft/MLOS/main/mlos_bench/mlos_bench/config/schemas/tunables/tunable-values-schema.json",
 
- "sched_migration_cost_ns": 500000,
- "sched_latency_ns": 12000000,
+ // Values that are different from the defaults
+ // in order to test --tunable-values handling in OneShotOptimizer.
+ "sched_migration_cost_ns": 400000,
+ "sched_latency_ns": 10000000,
  "sched_child_runs_first": "0",
  "sched_tunable_scaling": "1"
 }

mlos_bench/mlos_bench/tests/conftest.py

@@ -40,9 +40,9 @@ def mock_env(tunable_groups: TunableGroups) -> MockEnv:
  name="Test Env",
  config={
  "tunable_params": ["provision", "boot", "kernel"],
- "seed": SEED,
- "range": [60, 120],
- "metrics": ["score"],
+ "mock_env_seed": SEED,
+ "mock_env_range": [60, 120],
+ "mock_env_metrics": ["score"],
  },
  tunables=tunable_groups
  )
@@ -57,8 +57,9 @@ def mock_env_no_noise(tunable_groups: TunableGroups) -> MockEnv:
  name="Test Env No Noise",
  config={
  "tunable_params": ["provision", "boot", "kernel"],
- "range": [60, 120],
- "metrics": ["score", "other_score"],
+ "mock_env_seed": -1,
+ "mock_env_range": [60, 120],
+ "mock_env_metrics": ["score", "other_score"],
  },
  tunables=tunable_groups
  )

mlos_bench/mlos_bench/tests/environments/composite_env_test.py

@@ -41,8 +41,8 @@ def composite_env(tunable_groups: TunableGroups) -> CompositeEnv:
  "EnvId": 1,
  },
  "required_args": ["vmName", "someConst", "global_param"],
- "range": [60, 120],
- "metrics": ["score"],
+ "mock_env_range": [60, 120],
+ "mock_env_metrics": ["score"],
  }
  },
  {
@@ -56,8 +56,8 @@ def composite_env(tunable_groups: TunableGroups) -> CompositeEnv:
  "global_param": "local"
  },
  "required_args": ["vmName"],
- "range": [60, 120],
- "metrics": ["score"],
+ "mock_env_range": [60, 120],
+ "mock_env_metrics": ["score"],
  }
  },
  {
@@ -70,8 +70,8 @@ def composite_env(tunable_groups: TunableGroups) -> CompositeEnv:
  "EnvId": 3,
  },
  "required_args": ["vmName", "vm_server_name", "vm_client_name"],
- "range": [60, 120],
- "metrics": ["score"],
+ "mock_env_range": [60, 120],
+ "mock_env_metrics": ["score"],
  }
  }
  ]
@@ -193,8 +193,8 @@ def nested_composite_env(tunable_groups: TunableGroups) -> CompositeEnv:
  "vm_server_name",
  "global_param"
  ],
- "range": [60, 120],
- "metrics": ["score"],
+ "mock_env_range": [60, 120],
+ "mock_env_metrics": ["score"],
  }
  },
  # ...
@@ -218,8 +218,8 @@ def nested_composite_env(tunable_groups: TunableGroups) -> CompositeEnv:
  "config": {
  "tunable_params": ["boot"],
  "required_args": ["vmName", "EnvId", "vm_client_name"],
- "range": [60, 120],
- "metrics": ["score"],
+ "mock_env_range": [60, 120],
+ "mock_env_metrics": ["score"],
  }
  },
  # ...

mlos_bench/mlos_bench/tests/launcher_in_process_test.py

@@ -17,11 +17,14 @@
  ("argv", "expected_score"), [
  ([
  "--config", "mlos_bench/mlos_bench/tests/config/cli/mock-bench.jsonc",
- ], 65.6742),
+ "--trial_config_repeat_count", "5",
+ "--mock_env_seed", "-1", # Deterministic Mock Environment.
+ ], 67.40329),
  ([
  "--config", "mlos_bench/mlos_bench/tests/config/cli/mock-opt.jsonc",
  "--trial_config_repeat_count", "3",
  "--max_suggestions", "3",
+ "--mock_env_seed", "42", # Noisy Mock Environment.
  ], 64.53897),
  ]
 )

mlos_bench/mlos_bench/tests/launcher_run_test.py

@@ -52,7 +52,9 @@ def _launch_main_app(root_path: str, local_exec_service: LocalExecService,
  # temp_dir = '/tmp'
  log_path = path_join(temp_dir, "mock-test.log")
  (return_code, _stdout, _stderr) = local_exec_service.local_exec(
- [f"./mlos_bench/mlos_bench/run.py {cli_config} --log_file '{log_path}'"],
+ ["./mlos_bench/mlos_bench/run.py" +
+ " --config_path ./mlos_bench/mlos_bench/tests/config/" +
+ f" {cli_config} --log_file '{log_path}'"],
  cwd=root_path)
  assert return_code == 0
 
@@ -74,14 +76,35 @@ def _launch_main_app(root_path: str, local_exec_service: LocalExecService,
 def test_launch_main_app_bench(root_path: str, local_exec_service: LocalExecService) -> None:
  """
  Run mlos_bench command-line application with mock benchmark config
- and check the results in the log.
+ and default tunable values and check the results in the log.
+ """
+ _launch_main_app(
+ root_path, local_exec_service,
+ " --config cli/mock-bench.jsonc" +
+ " --trial_config_repeat_count 5" +
+ " --mock_env_seed -1", # Deterministic Mock Environment.
+ [
+ f"^{_RE_DATE} run\\.py:\\d+ " +
+ r"_main INFO Final score: \{'score': 67\.40\d+\}\s*$",
+ ]
+ )
+
+
+def test_launch_main_app_bench_values(
+ root_path: str, local_exec_service: LocalExecService) -> None:
+ """
+ Run mlos_bench command-line application with mock benchmark config
+ and user-specified tunable values and check the results in the log.
  """
  _launch_main_app(
  root_path, local_exec_service,
- "--config mlos_bench/mlos_bench/tests/config/cli/mock-bench.jsonc",
+ " --config cli/mock-bench.jsonc" +
+ " --tunable_values tunable-values/tunable-values-example.jsonc" +
+ " --trial_config_repeat_count 5" +
+ " --mock_env_seed -1", # Deterministic Mock Environment.
  [
  f"^{_RE_DATE} run\\.py:\\d+ " +
- r"_main INFO Final score: \{'score': 65\.67\d+\}\s*$",
+ r"_main INFO Final score: \{'score': 67\.11\d+\}\s*$",
  ]
  )
 
@@ -93,7 +116,10 @@ def test_launch_main_app_opt(root_path: str, local_exec_service: LocalExecServic
  """
  _launch_main_app(
  root_path, local_exec_service,
- "--config mlos_bench/mlos_bench/tests/config/cli/mock-opt.jsonc --trial_config_repeat_count 3 --max_suggestions 3",
+ "--config cli/mock-opt.jsonc" +
+ " --trial_config_repeat_count 3" +
+ " --max_suggestions 3" +
+ " --mock_env_seed 42", # Noisy Mock Environment.
  [
  # Iteration 1: Expect first value to be the baseline
  f"^{_RE_DATE} mlos_core_optimizer\\.py:\\d+ " +