Merge branch 'main' into multi-fidelity

.devcontainer/devcontainer.json

@@ -71,8 +71,9 @@
  "lextudio.restructuredtext",
  "matangover.mypy",
  "ms-azuretools.vscode-docker",
- "ms-python.autopep8",
- "ms-python.flake8",
+ // TODO: Enable additional formatter extensions:
+ //"ms-python.black-formatter",
+ //"ms-python.isort",
  "ms-python.pylint",
  "ms-python.python",
  "ms-python.vscode-pylance",

.vscode/extensions.json

@@ -14,8 +14,9 @@
  "lextudio.restructuredtext",
  "matangover.mypy",
  "ms-azuretools.vscode-docker",
- "ms-python.autopep8",
- "ms-python.flake8",
+ // TODO: Enable additional formatter extensions:
+ //"ms-python.black-formatter",
+ //"ms-python.isort",
  "ms-python.pylint",
  "ms-python.python",
  "ms-python.vscode-pylance",

.vscode/settings.json

@@ -129,9 +129,10 @@
  "--experimental"
  ],
  "[python]": {
- "editor.defaultFormatter": "ms-python.autopep8",
- "editor.formatOnSave": true,
- "editor.formatOnSaveMode": "modifications"
+ // TODO: Enable black formatter
+ //"editor.defaultFormatter": "ms-python.black-formatter",
+ //"editor.formatOnSave": true,
+ //"editor.formatOnSaveMode": "modifications"
  },
  // See Also .vscode/launch.json for environment variable args to pytest during debug sessions.
  // For the rest, see setup.cfg

Makefile

@@ -69,7 +69,9 @@ ifneq (,$(filter format,$(MAKECMDGOALS)))
 endif
 
 build/format.${CONDA_ENV_NAME}.build-stamp: build/licenseheaders.${CONDA_ENV_NAME}.build-stamp
-# TODO: add isort and black formatters
+# TODO: enable isort and black formatters
+#build/format.${CONDA_ENV_NAME}.build-stamp: build/isort.${CONDA_ENV_NAME}.build-stamp
+#build/format.${CONDA_ENV_NAME}.build-stamp: build/black.${CONDA_ENV_NAME}.build-stamp
 build/format.${CONDA_ENV_NAME}.build-stamp:
  touch $@
 
@@ -93,10 +95,117 @@ build/licenseheaders.${CONDA_ENV_NAME}.build-stamp:
  -x mlos_bench/setup.py mlos_core/setup.py mlos_viz/setup.py
  touch $@
 
+.PHONY: isort
+isort: build/isort.${CONDA_ENV_NAME}.build-stamp
+
+ifneq (,$(filter isort,$(MAKECMDGOALS)))
+ FORMAT_PREREQS += build/isort.${CONDA_ENV_NAME}.build-stamp
+endif
+
+build/isort.${CONDA_ENV_NAME}.build-stamp: build/isort.mlos_core.${CONDA_ENV_NAME}.build-stamp
+build/isort.${CONDA_ENV_NAME}.build-stamp: build/isort.mlos_bench.${CONDA_ENV_NAME}.build-stamp
+build/isort.${CONDA_ENV_NAME}.build-stamp: build/isort.mlos_viz.${CONDA_ENV_NAME}.build-stamp
+build/isort.${CONDA_ENV_NAME}.build-stamp:
+ touch $@
+
+# NOTE: when using pattern rules (involving %) we can only add one line of
+# prerequisities, so we use this pattern to compose the list as variables.
+
+# Both isort and licenseheaders alter files, so only run one at a time, by
+# making licenseheaders an order-only prerequisite.
+ISORT_COMMON_PREREQS :=
+ifneq (,$(filter format licenseheaders,$(MAKECMDGOALS)))
+ISORT_COMMON_PREREQS += build/licenseheaders.${CONDA_ENV_NAME}.build-stamp
+endif
+ISORT_COMMON_PREREQS += build/conda-env.${CONDA_ENV_NAME}.build-stamp
+ISORT_COMMON_PREREQS += $(MLOS_GLOBAL_CONF_FILES)
+
+build/isort.mlos_core.${CONDA_ENV_NAME}.build-stamp: $(MLOS_CORE_PYTHON_FILES)
+build/isort.mlos_bench.${CONDA_ENV_NAME}.build-stamp: $(MLOS_BENCH_PYTHON_FILES)
+build/isort.mlos_viz.${CONDA_ENV_NAME}.build-stamp: $(MLOS_VIZ_PYTHON_FILES)
+
+build/isort.%.${CONDA_ENV_NAME}.build-stamp: $(ISORT_COMMON_PREREQS)
+ # Reformat python file imports with isort.
+ conda run -n ${CONDA_ENV_NAME} isort --verbose --only-modified --atomic -j0 $(filter %.py,$?)
+ touch $@
+
+.PHONY: black
+black: build/black.${CONDA_ENV_NAME}.build-stamp
+
+ifneq (,$(filter black,$(MAKECMDGOALS)))
+ FORMAT_PREREQS += build/black.${CONDA_ENV_NAME}.build-stamp
+endif
+
+build/black.${CONDA_ENV_NAME}.build-stamp: build/black.mlos_core.${CONDA_ENV_NAME}.build-stamp
+build/black.${CONDA_ENV_NAME}.build-stamp: build/black.mlos_bench.${CONDA_ENV_NAME}.build-stamp
+build/black.${CONDA_ENV_NAME}.build-stamp: build/black.mlos_viz.${CONDA_ENV_NAME}.build-stamp
+build/black.${CONDA_ENV_NAME}.build-stamp:
+ touch $@
+
+# Both black, licenseheaders, and isort all alter files, so only run one at a time, by
+# making licenseheaders and isort an order-only prerequisite.
+BLACK_COMMON_PREREQS :=
+ifneq (,$(filter format licenseheaders,$(MAKECMDGOALS)))
+BLACK_COMMON_PREREQS += build/licenseheaders.${CONDA_ENV_NAME}.build-stamp
+endif
+ifneq (,$(filter format isort,$(MAKECMDGOALS)))
+BLACK_COMMON_PREREQS += build/isort.${CONDA_ENV_NAME}.build-stamp
+endif
+BLACK_COMMON_PREREQS += build/conda-env.${CONDA_ENV_NAME}.build-stamp
+BLACK_COMMON_PREREQS += $(MLOS_GLOBAL_CONF_FILES)
+
+build/black.mlos_core.${CONDA_ENV_NAME}.build-stamp: $(MLOS_CORE_PYTHON_FILES)
+build/black.mlos_bench.${CONDA_ENV_NAME}.build-stamp: $(MLOS_BENCH_PYTHON_FILES)
+build/black.mlos_viz.${CONDA_ENV_NAME}.build-stamp: $(MLOS_VIZ_PYTHON_FILES)
+
+build/black.%.${CONDA_ENV_NAME}.build-stamp: $(BLACK_COMMON_PREREQS)
+ # Reformat python files with black.
+ conda run -n ${CONDA_ENV_NAME} black $(filter %.py,$?)
+ touch $@
 
 .PHONY: check
 check: pycodestyle pydocstyle pylint mypy # cspell markdown-link-check
-# TODO: add isort and black checks
+# TODO: Enable isort and black checks
+#check: isort-check black-check pycodestyle pydocstyle pylint mypy # cspell markdown-link-check
+
+.PHONY: black-check
+black-check: build/black-check.mlos_core.${CONDA_ENV_NAME}.build-stamp
+black-check: build/black-check.mlos_bench.${CONDA_ENV_NAME}.build-stamp
+black-check: build/black-check.mlos_viz.${CONDA_ENV_NAME}.build-stamp
+
+# Make sure black format rules run before black-check rules.
+build/black-check.mlos_core.${CONDA_ENV_NAME}.build-stamp: $(MLOS_CORE_PYTHON_FILES)
+build/black-check.mlos_bench.${CONDA_ENV_NAME}.build-stamp: $(MLOS_BENCH_PYTHON_FILES)
+build/black-check.mlos_viz.${CONDA_ENV_NAME}.build-stamp: $(MLOS_VIZ_PYTHON_FILES)
+
+BLACK_CHECK_COMMON_PREREQS := build/conda-env.${CONDA_ENV_NAME}.build-stamp
+BLACK_CHECK_COMMON_PREREQS += $(FORMAT_PREREQS)
+BLACK_CHECK_COMMON_PREREQS += $(MLOS_GLOBAL_CONF_FILES)
+
+build/black-check.%.${CONDA_ENV_NAME}.build-stamp: $(BLACK_CHECK_COMMON_PREREQS)
+ # Check for import sort order.
+ # Note: if this fails use "make format" or "make black" to fix it.
+ conda run -n ${CONDA_ENV_NAME} black --verbose --check --diff $(filter %.py,$?)
+ touch $@
+
+.PHONY: isort-check
+isort-check: build/isort-check.mlos_core.${CONDA_ENV_NAME}.build-stamp
+isort-check: build/isort-check.mlos_bench.${CONDA_ENV_NAME}.build-stamp
+isort-check: build/isort-check.mlos_viz.${CONDA_ENV_NAME}.build-stamp
+
+# Make sure isort format rules run before isort-check rules.
+build/isort-check.mlos_core.${CONDA_ENV_NAME}.build-stamp: $(MLOS_CORE_PYTHON_FILES)
+build/isort-check.mlos_bench.${CONDA_ENV_NAME}.build-stamp: $(MLOS_BENCH_PYTHON_FILES)
+build/isort-check.mlos_viz.${CONDA_ENV_NAME}.build-stamp: $(MLOS_VIZ_PYTHON_FILES)
+
+ISORT_CHECK_COMMON_PREREQS := build/conda-env.${CONDA_ENV_NAME}.build-stamp
+ISORT_CHECK_COMMON_PREREQS += $(FORMAT_PREREQS)
+ISORT_CHECK_COMMON_PREREQS += $(MLOS_GLOBAL_CONF_FILES)
+
+build/isort-check.%.${CONDA_ENV_NAME}.build-stamp: $(ISORT_CHECK_COMMON_PREREQS)
+ # Note: if this fails use "make format" or "make isort" to fix it.
+ conda run -n ${CONDA_ENV_NAME} isort --only-modified --check --diff -j0 $(filter %.py,$?)
+ touch $@
 
 .PHONY: pycodestyle
 pycodestyle: build/pycodestyle.mlos_core.${CONDA_ENV_NAME}.build-stamp

conda-envs/mlos-3.10.yml

@@ -7,6 +7,7 @@ dependencies:
  # All other dependencies for the mlos modules come from pip.
  - pip
  - pylint
+ - black
  - pycodestyle
  - pydocstyle
  - flake8
@@ -27,6 +28,7 @@ dependencies:
  - autopep8>=1.7.0
  - bump2version
  - check-jsonschema
+ - isort
  - licenseheaders
  - mypy
  - pandas-stubs
@@ -36,6 +38,9 @@ dependencies:
  - types-pygments
  - types-requests
  - types-setuptools
- - "--editable ../mlos_core[full-tests]"
- - "--editable ../mlos_bench[full-tests]"
- - "--editable ../mlos_viz[full-tests]"
+ # Workaround a pylance issue in vscode that prevents it finding the latest
+ # method of pip installing editable modules.
+ # https://github.com/microsoft/pylance-release/issues/3473
+ - "--config-settings editable_mode=compat --editable ../mlos_core[full-tests]"
+ - "--config-settings editable_mode=compat --editable ../mlos_bench[full-tests]"
+ - "--config-settings editable_mode=compat --editable ../mlos_viz[full-tests]"

conda-envs/mlos-3.11.yml

@@ -7,6 +7,7 @@ dependencies:
  # All other dependencies for the mlos modules come from pip.
  - pip
  - pylint
+ - black
  - pycodestyle
  - pydocstyle
  - flake8
@@ -27,6 +28,7 @@ dependencies:
  - autopep8>=1.7.0
  - bump2version
  - check-jsonschema
+ - isort
  - licenseheaders
  - mypy
  - pandas-stubs
@@ -36,6 +38,9 @@ dependencies:
  - types-pygments
  - types-requests
  - types-setuptools
- - "--editable ../mlos_core[full-tests]"
- - "--editable ../mlos_bench[full-tests]"
- - "--editable ../mlos_viz[full-tests]"
+ # Workaround a pylance issue in vscode that prevents it finding the latest
+ # method of pip installing editable modules.
+ # https://github.com/microsoft/pylance-release/issues/3473
+ - "--config-settings editable_mode=compat --editable ../mlos_core[full-tests]"
+ - "--config-settings editable_mode=compat --editable ../mlos_bench[full-tests]"
+ - "--config-settings editable_mode=compat --editable ../mlos_viz[full-tests]"

conda-envs/mlos-3.8.yml

@@ -7,6 +7,7 @@ dependencies:
  # All other dependencies for the mlos modules come from pip.
  - pip
  - pylint
+ - black
  - pycodestyle
  - pydocstyle
  - flake8
@@ -27,6 +28,7 @@ dependencies:
  - autopep8>=1.7.0
  - bump2version
  - check-jsonschema
+ - isort
  - licenseheaders
  - mypy
  - pandas-stubs
@@ -36,6 +38,9 @@ dependencies:
  - types-pygments
  - types-requests
  - types-setuptools
- - "--editable ../mlos_core[full-tests]"
- - "--editable ../mlos_bench[full-tests]"
- - "--editable ../mlos_viz[full-tests]"
+ # Workaround a pylance issue in vscode that prevents it finding the latest
+ # method of pip installing editable modules.
+ # https://github.com/microsoft/pylance-release/issues/3473
+ - "--config-settings editable_mode=compat --editable ../mlos_core[full-tests]"
+ - "--config-settings editable_mode=compat --editable ../mlos_bench[full-tests]"
+ - "--config-settings editable_mode=compat --editable ../mlos_viz[full-tests]"

conda-envs/mlos-3.9.yml

@@ -7,6 +7,7 @@ dependencies:
  # All other dependencies for the mlos modules come from pip.
  - pip
  - pylint
+ - black
  - pycodestyle
  - pydocstyle
  - flake8
@@ -27,6 +28,7 @@ dependencies:
  - autopep8>=1.7.0
  - bump2version
  - check-jsonschema
+ - isort
  - licenseheaders
  - mypy
  - pandas-stubs
@@ -36,6 +38,9 @@ dependencies:
  - types-pygments
  - types-requests
  - types-setuptools
- - "--editable ../mlos_core[full-tests]"
- - "--editable ../mlos_bench[full-tests]"
- - "--editable ../mlos_viz[full-tests]"
+ # Workaround a pylance issue in vscode that prevents it finding the latest
+ # method of pip installing editable modules.
+ # https://github.com/microsoft/pylance-release/issues/3473
+ - "--config-settings editable_mode=compat --editable ../mlos_core[full-tests]"
+ - "--config-settings editable_mode=compat --editable ../mlos_bench[full-tests]"
+ - "--config-settings editable_mode=compat --editable ../mlos_viz[full-tests]"

conda-envs/mlos-windows.yml

@@ -8,6 +8,7 @@ dependencies:
  # All other dependencies for the mlos modules come from pip.
  - pip
  - pylint
+ - black
  - pycodestyle
  - pydocstyle
  - flake8
@@ -30,6 +31,7 @@ dependencies:
  - autopep8>=1.7.0
  - bump2version
  - check-jsonschema
+ - isort
  - licenseheaders
  - mypy
  - pandas-stubs
@@ -40,6 +42,9 @@ dependencies:
  - types-requests
  - types-setuptools
  - pyarrow
- - "--editable ../mlos_core[full-tests]"
- - "--editable ../mlos_bench[full-tests]"
- - "--editable ../mlos_viz[full-tests]"
+ # Workaround a pylance issue in vscode that prevents it finding the latest
+ # method of pip installing editable modules.
+ # https://github.com/microsoft/pylance-release/issues/3473
+ - "--config-settings editable_mode=compat --editable ../mlos_core[full-tests]"
+ - "--config-settings editable_mode=compat --editable ../mlos_bench[full-tests]"
+ - "--config-settings editable_mode=compat --editable ../mlos_viz[full-tests]"

conda-envs/mlos.yml

@@ -7,6 +7,7 @@ dependencies:
  # All other dependencies for the mlos modules come from pip.
  - pip
  - pylint
+ - black
  - pycodestyle
  - pydocstyle
  - flake8
@@ -26,6 +27,7 @@ dependencies:
  - autopep8>=1.7.0
  - bump2version
  - check-jsonschema
+ - isort
  - licenseheaders
  - mypy
  - pandas-stubs
@@ -35,6 +37,9 @@ dependencies:
  - types-pygments
  - types-requests
  - types-setuptools
- - "--editable ../mlos_core[full-tests]"
- - "--editable ../mlos_bench[full-tests]"
- - "--editable ../mlos_viz[full-tests]"
+ # Workaround a pylance issue in vscode that prevents it finding the latest
+ # method of pip installing editable modules.
+ # https://github.com/microsoft/pylance-release/issues/3473
+ - "--config-settings editable_mode=compat --editable ../mlos_core[full-tests]"
+ - "--config-settings editable_mode=compat --editable ../mlos_bench[full-tests]"
+ - "--config-settings editable_mode=compat --editable ../mlos_viz[full-tests]"

mlos_bench/mlos_bench/optimizers/one_shot_optimizer.py

@@ -18,7 +18,7 @@
 
 class OneShotOptimizer(MockOptimizer):
  """
- Mock optimizer that proposes a single configuration and returns.
+ No-op optimizer that proposes a single configuration and returns.
  Explicit configs (partial or full) are possible using configuration files.
  """
 
@@ -33,6 +33,14 @@ def __init__(self,
  _LOG.info("Run a single iteration for: %s", self._tunables)
  self._max_iter = 1 # Always run for just one iteration.
 
+ def suggest(self) -> TunableGroups:
+ """
+ Always produce the same (initial) suggestion.
+ """
+ tunables = super().suggest()
+ self._start_with_defaults = True
+ return tunables
+
  @property
  def supports_preload(self) -> bool:
  return False

mlos_bench/mlos_bench/schedulers/base_scheduler.py

@@ -215,7 +215,7 @@ def schedule_trial(self, tunables: TunableGroups) -> None:
  # prevented).
  "optimizer": self.optimizer.name,
  "repeat_i": repeat_i,
- "is_defaults": tunables.is_defaults,
+ "is_defaults": tunables.is_defaults(),
  **{
  f"opt_{key}_{i}": val
  for (i, opt_target) in enumerate(self.optimizer.targets.items())

mlos_bench/mlos_bench/storage/sql/experiment.py

@@ -253,7 +253,9 @@ def _get_config_id(self, conn: Connection, tunables: TunableGroups) -> int:
 
  def new_trial(self, tunables: TunableGroups, ts_start: Optional[datetime] = None,
  config: Optional[Dict[str, Any]] = None) -> Storage.Trial:
- ts_start = utcify_timestamp(ts_start or datetime.now(UTC), origin="local")
+ # MySQL can round microseconds into the future causing scheduler to skip trials.
+ # Truncate microseconds to avoid this issue.
+ ts_start = utcify_timestamp(ts_start or datetime.now(UTC), origin="local").replace(microsecond=0)
  _LOG.debug("Create trial: %s:%d @ %s", self._experiment_id, self._trial_id, ts_start)
  with self._engine.begin() as conn:
  try:

mlos_bench/pyproject.toml

@@ -1,5 +1,5 @@
 [build-system]
-requires = ["setuptools", "setuptools-scm>=8.1.0", "wheel"]
+requires = ["setuptools>64", "setuptools-scm>=8.1.0", "wheel"]
 build-backend = "setuptools.build_meta"
 
 [project]