Prepare isort black formatters and checks (#766)

Builds off of #762, #763, and #764.

Prepares rules and configs to enable isort and black formatters and
checks but doesn't enable them yet.

After these are enabled (next PR) we will reformat all files and ignore
that revision in git blame configs.

Then, we can convert configs stored in `setup.cfg` and `.pylintrc` to
the top level `pyproject.toml` and remove the older configs.

.devcontainer/devcontainer.json

@@ -71,8 +71,9 @@
  "lextudio.restructuredtext",
  "matangover.mypy",
  "ms-azuretools.vscode-docker",
- "ms-python.autopep8",
- "ms-python.flake8",
+ // TODO: Enable additional formatter extensions:
+ //"ms-python.black-formatter",
+ //"ms-python.isort",
  "ms-python.pylint",
  "ms-python.python",
  "ms-python.vscode-pylance",

.vscode/extensions.json

@@ -14,8 +14,9 @@
  "lextudio.restructuredtext",
  "matangover.mypy",
  "ms-azuretools.vscode-docker",
- "ms-python.autopep8",
- "ms-python.flake8",
+ // TODO: Enable additional formatter extensions:
+ //"ms-python.black-formatter",
+ //"ms-python.isort",
  "ms-python.pylint",
  "ms-python.python",
  "ms-python.vscode-pylance",

.vscode/settings.json

@@ -129,9 +129,10 @@
  "--experimental"
  ],
  "[python]": {
- "editor.defaultFormatter": "ms-python.autopep8",
- "editor.formatOnSave": true,
- "editor.formatOnSaveMode": "modifications"
+ // TODO: Enable black formatter
+ //"editor.defaultFormatter": "ms-python.black-formatter",
+ //"editor.formatOnSave": true,
+ //"editor.formatOnSaveMode": "modifications"
  },
  // See Also .vscode/launch.json for environment variable args to pytest during debug sessions.
  // For the rest, see setup.cfg

Makefile

@@ -69,7 +69,9 @@ ifneq (,$(filter format,$(MAKECMDGOALS)))
 endif
 
 build/format.${CONDA_ENV_NAME}.build-stamp: build/licenseheaders.${CONDA_ENV_NAME}.build-stamp
-# TODO: add isort and black formatters
+# TODO: enable isort and black formatters
+#build/format.${CONDA_ENV_NAME}.build-stamp: build/isort.${CONDA_ENV_NAME}.build-stamp
+#build/format.${CONDA_ENV_NAME}.build-stamp: build/black.${CONDA_ENV_NAME}.build-stamp
 build/format.${CONDA_ENV_NAME}.build-stamp:
  touch $@
 
@@ -93,10 +95,117 @@ build/licenseheaders.${CONDA_ENV_NAME}.build-stamp:
  -x mlos_bench/setup.py mlos_core/setup.py mlos_viz/setup.py
  touch $@
 
+.PHONY: isort
+isort: build/isort.${CONDA_ENV_NAME}.build-stamp
+
+ifneq (,$(filter isort,$(MAKECMDGOALS)))
+ FORMAT_PREREQS += build/isort.${CONDA_ENV_NAME}.build-stamp
+endif
+
+build/isort.${CONDA_ENV_NAME}.build-stamp: build/isort.mlos_core.${CONDA_ENV_NAME}.build-stamp
+build/isort.${CONDA_ENV_NAME}.build-stamp: build/isort.mlos_bench.${CONDA_ENV_NAME}.build-stamp
+build/isort.${CONDA_ENV_NAME}.build-stamp: build/isort.mlos_viz.${CONDA_ENV_NAME}.build-stamp
+build/isort.${CONDA_ENV_NAME}.build-stamp:
+ touch $@
+
+# NOTE: when using pattern rules (involving %) we can only add one line of
+# prerequisities, so we use this pattern to compose the list as variables.
+
+# Both isort and licenseheaders alter files, so only run one at a time, by
+# making licenseheaders an order-only prerequisite.
+ISORT_COMMON_PREREQS :=
+ifneq (,$(filter format licenseheaders,$(MAKECMDGOALS)))
+ISORT_COMMON_PREREQS += build/licenseheaders.${CONDA_ENV_NAME}.build-stamp
+endif
+ISORT_COMMON_PREREQS += build/conda-env.${CONDA_ENV_NAME}.build-stamp
+ISORT_COMMON_PREREQS += $(MLOS_GLOBAL_CONF_FILES)
+
+build/isort.mlos_core.${CONDA_ENV_NAME}.build-stamp: $(MLOS_CORE_PYTHON_FILES)
+build/isort.mlos_bench.${CONDA_ENV_NAME}.build-stamp: $(MLOS_BENCH_PYTHON_FILES)
+build/isort.mlos_viz.${CONDA_ENV_NAME}.build-stamp: $(MLOS_VIZ_PYTHON_FILES)
+
+build/isort.%.${CONDA_ENV_NAME}.build-stamp: $(ISORT_COMMON_PREREQS)
+ # Reformat python file imports with isort.
+ conda run -n ${CONDA_ENV_NAME} isort --verbose --only-modified --atomic -j0 $(filter %.py,$?)
+ touch $@
+
+.PHONY: black
+black: build/black.${CONDA_ENV_NAME}.build-stamp
+
+ifneq (,$(filter black,$(MAKECMDGOALS)))
+ FORMAT_PREREQS += build/black.${CONDA_ENV_NAME}.build-stamp
+endif
+
+build/black.${CONDA_ENV_NAME}.build-stamp: build/black.mlos_core.${CONDA_ENV_NAME}.build-stamp
+build/black.${CONDA_ENV_NAME}.build-stamp: build/black.mlos_bench.${CONDA_ENV_NAME}.build-stamp
+build/black.${CONDA_ENV_NAME}.build-stamp: build/black.mlos_viz.${CONDA_ENV_NAME}.build-stamp
+build/black.${CONDA_ENV_NAME}.build-stamp:
+ touch $@
+
+# Both black, licenseheaders, and isort all alter files, so only run one at a time, by
+# making licenseheaders and isort an order-only prerequisite.
+BLACK_COMMON_PREREQS :=
+ifneq (,$(filter format licenseheaders,$(MAKECMDGOALS)))
+BLACK_COMMON_PREREQS += build/licenseheaders.${CONDA_ENV_NAME}.build-stamp
+endif
+ifneq (,$(filter format isort,$(MAKECMDGOALS)))
+BLACK_COMMON_PREREQS += build/isort.${CONDA_ENV_NAME}.build-stamp
+endif
+BLACK_COMMON_PREREQS += build/conda-env.${CONDA_ENV_NAME}.build-stamp
+BLACK_COMMON_PREREQS += $(MLOS_GLOBAL_CONF_FILES)
+
+build/black.mlos_core.${CONDA_ENV_NAME}.build-stamp: $(MLOS_CORE_PYTHON_FILES)
+build/black.mlos_bench.${CONDA_ENV_NAME}.build-stamp: $(MLOS_BENCH_PYTHON_FILES)
+build/black.mlos_viz.${CONDA_ENV_NAME}.build-stamp: $(MLOS_VIZ_PYTHON_FILES)
+
+build/black.%.${CONDA_ENV_NAME}.build-stamp: $(BLACK_COMMON_PREREQS)
+ # Reformat python files with black.
+ conda run -n ${CONDA_ENV_NAME} black $(filter %.py,$?)
+ touch $@
 
 .PHONY: check
 check: pycodestyle pydocstyle pylint mypy # cspell markdown-link-check
-# TODO: add isort and black checks
+# TODO: Enable isort and black checks
+#check: isort-check black-check pycodestyle pydocstyle pylint mypy # cspell markdown-link-check
+
+.PHONY: black-check
+black-check: build/black-check.mlos_core.${CONDA_ENV_NAME}.build-stamp
+black-check: build/black-check.mlos_bench.${CONDA_ENV_NAME}.build-stamp
+black-check: build/black-check.mlos_viz.${CONDA_ENV_NAME}.build-stamp
+
+# Make sure black format rules run before black-check rules.
+build/black-check.mlos_core.${CONDA_ENV_NAME}.build-stamp: $(MLOS_CORE_PYTHON_FILES)
+build/black-check.mlos_bench.${CONDA_ENV_NAME}.build-stamp: $(MLOS_BENCH_PYTHON_FILES)
+build/black-check.mlos_viz.${CONDA_ENV_NAME}.build-stamp: $(MLOS_VIZ_PYTHON_FILES)
+
+BLACK_CHECK_COMMON_PREREQS := build/conda-env.${CONDA_ENV_NAME}.build-stamp
+BLACK_CHECK_COMMON_PREREQS += $(FORMAT_PREREQS)
+BLACK_CHECK_COMMON_PREREQS += $(MLOS_GLOBAL_CONF_FILES)
+
+build/black-check.%.${CONDA_ENV_NAME}.build-stamp: $(BLACK_CHECK_COMMON_PREREQS)
+ # Check for import sort order.
+ # Note: if this fails use "make format" or "make black" to fix it.
+ conda run -n ${CONDA_ENV_NAME} black --verbose --check --diff $(filter %.py,$?)
+ touch $@
+
+.PHONY: isort-check
+isort-check: build/isort-check.mlos_core.${CONDA_ENV_NAME}.build-stamp
+isort-check: build/isort-check.mlos_bench.${CONDA_ENV_NAME}.build-stamp
+isort-check: build/isort-check.mlos_viz.${CONDA_ENV_NAME}.build-stamp
+
+# Make sure isort format rules run before isort-check rules.
+build/isort-check.mlos_core.${CONDA_ENV_NAME}.build-stamp: $(MLOS_CORE_PYTHON_FILES)
+build/isort-check.mlos_bench.${CONDA_ENV_NAME}.build-stamp: $(MLOS_BENCH_PYTHON_FILES)
+build/isort-check.mlos_viz.${CONDA_ENV_NAME}.build-stamp: $(MLOS_VIZ_PYTHON_FILES)
+
+ISORT_CHECK_COMMON_PREREQS := build/conda-env.${CONDA_ENV_NAME}.build-stamp
+ISORT_CHECK_COMMON_PREREQS += $(FORMAT_PREREQS)
+ISORT_CHECK_COMMON_PREREQS += $(MLOS_GLOBAL_CONF_FILES)
+
+build/isort-check.%.${CONDA_ENV_NAME}.build-stamp: $(ISORT_CHECK_COMMON_PREREQS)
+ # Note: if this fails use "make format" or "make isort" to fix it.
+ conda run -n ${CONDA_ENV_NAME} isort --only-modified --check --diff -j0 $(filter %.py,$?)
+ touch $@
 
 .PHONY: pycodestyle
 pycodestyle: build/pycodestyle.mlos_core.${CONDA_ENV_NAME}.build-stamp

conda-envs/mlos-3.10.yml

@@ -7,6 +7,7 @@ dependencies:
  # All other dependencies for the mlos modules come from pip.
  - pip
  - pylint
+ - black
  - pycodestyle
  - pydocstyle
  - flake8
@@ -27,6 +28,7 @@ dependencies:
  - autopep8>=1.7.0
  - bump2version
  - check-jsonschema
+ - isort
  - licenseheaders
  - mypy
  - pandas-stubs

conda-envs/mlos-3.11.yml

@@ -7,6 +7,7 @@ dependencies:
  # All other dependencies for the mlos modules come from pip.
  - pip
  - pylint
+ - black
  - pycodestyle
  - pydocstyle
  - flake8
@@ -27,6 +28,7 @@ dependencies:
  - autopep8>=1.7.0
  - bump2version
  - check-jsonschema
+ - isort
  - licenseheaders
  - mypy
  - pandas-stubs

conda-envs/mlos-3.8.yml

@@ -7,6 +7,7 @@ dependencies:
  # All other dependencies for the mlos modules come from pip.
  - pip
  - pylint
+ - black
  - pycodestyle
  - pydocstyle
  - flake8
@@ -27,6 +28,7 @@ dependencies:
  - autopep8>=1.7.0
  - bump2version
  - check-jsonschema
+ - isort
  - licenseheaders
  - mypy
  - pandas-stubs

conda-envs/mlos-3.9.yml

@@ -7,6 +7,7 @@ dependencies:
  # All other dependencies for the mlos modules come from pip.
  - pip
  - pylint
+ - black
  - pycodestyle
  - pydocstyle
  - flake8
@@ -27,6 +28,7 @@ dependencies:
  - autopep8>=1.7.0
  - bump2version
  - check-jsonschema
+ - isort
  - licenseheaders
  - mypy
  - pandas-stubs

conda-envs/mlos-windows.yml

@@ -8,6 +8,7 @@ dependencies:
  # All other dependencies for the mlos modules come from pip.
  - pip
  - pylint
+ - black
  - pycodestyle
  - pydocstyle
  - flake8
@@ -30,6 +31,7 @@ dependencies:
  - autopep8>=1.7.0
  - bump2version
  - check-jsonschema
+ - isort
  - licenseheaders
  - mypy
  - pandas-stubs

conda-envs/mlos.yml

@@ -7,6 +7,7 @@ dependencies:
  # All other dependencies for the mlos modules come from pip.
  - pip
  - pylint
+ - black
  - pycodestyle
  - pydocstyle
  - flake8
@@ -26,6 +27,7 @@ dependencies:
  - autopep8>=1.7.0
  - bump2version
  - check-jsonschema
+ - isort
  - licenseheaders
  - mypy
  - pandas-stubs

pyproject.toml

@@ -0,0 +1,9 @@
+[tool.black]
+line-length = 88
+target-version = ["py38", "py39", "py310", "py311", "py312"]
+include = '\.pyi?$'
+
+[tool.isort]
+profile = "black"
+py_version = 311
+src_paths = ["mlos_core", "mlos_bench", "mlos_viz"]