From 36e204fb6bf69b6fb48d871be37906f906bd6fdc Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Mon, 15 May 2023 20:48:47 +0000 Subject: [PATCH 01/12] Disable line buffering --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a42cfb8573f..8d0945fcdd8 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ MKDIR_BUILD := $(shell test -d build || mkdir build) # Run make in parallel by default. MAKEFLAGS += -j$(shell nproc) -MAKEFLAGS += -Oline +#MAKEFLAGS += -Oline .PHONY: all all: check test dist dist-test doc licenseheaders From 242eecab8a7569aca3f724e293a4f59a2f1096aa Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Mon, 15 May 2023 20:21:55 +0000 Subject: [PATCH 02/12] fail pylint checks on unused-import errors --- .pylintrc | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.pylintrc b/.pylintrc index d1859603649..73b91f49ba3 100644 --- a/.pylintrc +++ b/.pylintrc @@ -6,7 +6,10 @@ fail-under=9.7 # Make sure public methods are documented. # See Also: https://github.com/PyCQA/pydocstyle/issues/309#issuecomment-1426642147 -fail-on=C0116 +# Also fail on unused imports. +fail-on= + C0116, + unused-import # Ignore pylint complaints about an upstream dependency. ignored-modules=ConfigSpace.hyperparameters From 901c513593a311adaf86260ddb5103d220b61a54 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Mon, 15 May 2023 20:23:28 +0000 Subject: [PATCH 03/12] make sure pycodestyle checks for indentation --- setup.cfg | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index d9b475d29a3..f6eb77d54aa 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,12 +5,11 @@ universal = 1 [pycodestyle] count = True # E124: Closing bracket does not match indentation of opening bracket's line -# E128: Continuation line under-indented for visual indent # E261: At least two spaces before inline comment # E502: The backslash is redundant between brackets # W503: Line break occurred before a binary operator # W504: Line break occurred after a binary operator -ignore = E124,E128,E261,E502,W503,W504 +ignore = E124,E261,E502,W503,W504 format = pylint # See Also: .editorconfig, .pylintrc max-line-length = 132 From ce8923a2102b087fd4e8e455d0a61663325ea216 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Mon, 15 May 2023 20:24:59 +0000 Subject: [PATCH 04/12] increase code quality requirements --- .pylintrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pylintrc b/.pylintrc index 73b91f49ba3..bb0ab42cd0d 100644 --- a/.pylintrc +++ b/.pylintrc @@ -2,7 +2,7 @@ [MAIN] # Specify a score threshold to be exceeded before program exits with error. -fail-under=9.7 +fail-under=9.8 # Make sure public methods are documented. # See Also: https://github.com/PyCQA/pydocstyle/issues/309#issuecomment-1426642147 From 0459ca87ce0006a7b16a4d2c0fb5bc6f09a8195f Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Mon, 15 May 2023 21:21:14 +0000 Subject: [PATCH 05/12] add some new extensions --- .devcontainer/devcontainer.json | 55 ++++++++++++++++++--------------- .vscode/extensions.json | 49 +++++++++++++++-------------- conda-envs/mlos-3.10.yml | 1 + conda-envs/mlos-3.11.yml | 1 + conda-envs/mlos-3.8.yml | 1 + conda-envs/mlos-3.9.yml | 1 + conda-envs/mlos-windows.yml | 1 + conda-envs/mlos.yml | 1 + 8 files changed, 62 insertions(+), 48 deletions(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 26e931c658a..903903a33e6 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -43,40 +43,45 @@ "python.defaultInterpreterPath": "/opt/conda/envs/mlos/bin/python", "python.testing.pytestPath": "/opt/conda/envs/mlos/bin/pytest", "python.linting.pylintPath": "/opt/conda/envs/mlos/bin/pylint", - "pylint.path": ["/opt/conda/envs/mlos/bin/pylint"], + "pylint.path": [ + "/opt/conda/envs/mlos/bin/pylint" + ], + "python.linting.flake8Path": "/opt/conda/envs/mlos/bin/flake8", "python.linting.mypyPath": "/opt/conda/envs/mlos/bin/mypy", "mypy.dmypyExecutable": "/opt/conda/envs/mlos/bin/dmypy", "mypy.runUsingActiveInterpreter": false }, // Add the IDs of extensions you want installed when the container is created. "extensions": [ - "EditorConfig.EditorConfig", - "streetsidesoftware.code-spell-checker", - "ms-python.vscode-pylance", - "ms-python.python", - "ms-python.pylint", - "matangover.mypy", + "davidanson.vscode-markdownlint", + "donjayamanne.githistory", "donjayamanne.python-environment-manager", - "njpwerner.autodocstring", - "ms-toolsai.jupyter", + "eamodio.gitlens", + "editorconfig.editorconfig", + "esbenp.prettier-vscode", + "github.copilot", + "github.vscode-github-actions", + "github.vscode-pull-request-github", + "gruntfuggly.todo-tree", + "hashicorp.terraform", + "huntertran.auto-markdown-toc", + "ibm.output-colorizer", "lextudio.restructuredtext", - "trond-snekvik.simple-rst", - "DavidAnson.vscode-markdownlint", // Linter for markdown files - "huntertran.auto-markdown-toc", // Auto-generated Markdown Table of Contents - "github.vscode-github-actions", // GitHub Actions integration + "matangover.mypy", "ms-azuretools.vscode-docker", - "waderyan.gitblame", // Enhances git blame experience - "donjayamanne.githistory", // Enhanced git history experience - "eamodio.gitlens", - "ms-vsliveshare.vsliveshare", // Allows easy code share - "Gruntfuggly.todo-tree", // Highlights TODO comments in code - "github.copilot", // Copilot integration - "IBM.output-colorizer", // Colorize your output/test logs - "hashicorp.terraform", // Terraform syntax highlighting - "redhat.vscode-yaml", // Kubernetes manifest syntax highlighting - "stkb.rewrap", // "Alt + Q" -> rewrap comments after n characters on one line - "tyriar.sort-lines", // "Ctrl + Shift + P -> Sort lines" -> sort lines alphabetically - "esbenp.prettier-vscode" // A pretty good formatter for many languages + "ms-python.autopep8", + "ms-python.flake8", + "ms-python.pylint", + "ms-python.python", + "ms-python.vscode-pylance", + "ms-vsliveshare.vsliveshare", + "njpwerner.autodocstring", + "redhat.vscode-yaml", + "stkb.rewrap", + "streetsidesoftware.code-spell-checker", + "trond-snekvik.simple-rst", + "tyriar.sort-lines", + "waderyan.gitblame" ] } } diff --git a/.vscode/extensions.json b/.vscode/extensions.json index 76055ecaddc..7c520200b82 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -1,30 +1,33 @@ { "recommendations": [ - "EditorConfig.EditorConfig", - "streetsidesoftware.code-spell-checker", - "ms-python.vscode-pylance", - "ms-python.python", - "ms-python.pylint", - "matangover.mypy", + "davidanson.vscode-markdownlint", + "donjayamanne.githistory", "donjayamanne.python-environment-manager", - "njpwerner.autodocstring", + "eamodio.gitlens", + "editorconfig.editorconfig", + "esbenp.prettier-vscode", + "github.copilot", + "github.vscode-github-actions", + "github.vscode-pull-request-github", + "gruntfuggly.todo-tree", + "hashicorp.terraform", + "huntertran.auto-markdown-toc", + "ibm.output-colorizer", "lextudio.restructuredtext", - "trond-snekvik.simple-rst", - "DavidAnson.vscode-markdownlint", // Linter for markdown files - "huntertran.auto-markdown-toc", // Auto-generated Markdown Table of Contents - "github.vscode-github-actions", // GitHub Actions integration + "matangover.mypy", "ms-azuretools.vscode-docker", - "waderyan.gitblame", // Enhances git blame experience - "donjayamanne.githistory", // Enhanced git history experience - "eamodio.gitlens", - "ms-vsliveshare.vsliveshare", // Allows easy code share - "Gruntfuggly.todo-tree", // Highlights TODO comments in code - "github.copilot", // Copilot integration - "IBM.output-colorizer", // Colorize your output/test logs - "hashicorp.terraform", // Terraform syntax highlighting - "redhat.vscode-yaml", // Kubernetes manifest syntax highlighting - "stkb.rewrap", // "Alt + Q" -> rewrap comments after n characters on one line - "tyriar.sort-lines", // "Ctrl + Shift + P -> Sort lines" -> sort lines alphabetically - "esbenp.prettier-vscode" // A pretty good formatter for many languages + "ms-python.autopep8", + "ms-python.flake8", + "ms-python.pylint", + "ms-python.python", + "ms-python.vscode-pylance", + "ms-vsliveshare.vsliveshare", + "njpwerner.autodocstring", + "redhat.vscode-yaml", + "stkb.rewrap", + "streetsidesoftware.code-spell-checker", + "trond-snekvik.simple-rst", + "tyriar.sort-lines", + "waderyan.gitblame" ] } diff --git a/conda-envs/mlos-3.10.yml b/conda-envs/mlos-3.10.yml index a99cad74ba1..4a219ad18fd 100644 --- a/conda-envs/mlos-3.10.yml +++ b/conda-envs/mlos-3.10.yml @@ -9,6 +9,7 @@ dependencies: - pycodestyle - autopep8 - pydocstyle + - flake8 - setuptools - setuptools-scm - jupyter diff --git a/conda-envs/mlos-3.11.yml b/conda-envs/mlos-3.11.yml index 828d9cc22c6..071d6a7c5bb 100644 --- a/conda-envs/mlos-3.11.yml +++ b/conda-envs/mlos-3.11.yml @@ -9,6 +9,7 @@ dependencies: - pycodestyle - autopep8 - pydocstyle + - flake8 - setuptools - setuptools-scm - jupyter diff --git a/conda-envs/mlos-3.8.yml b/conda-envs/mlos-3.8.yml index 844895594da..7f4a23d53a7 100644 --- a/conda-envs/mlos-3.8.yml +++ b/conda-envs/mlos-3.8.yml @@ -9,6 +9,7 @@ dependencies: - pycodestyle - autopep8 - pydocstyle + - flake8 - setuptools - setuptools-scm - jupyter diff --git a/conda-envs/mlos-3.9.yml b/conda-envs/mlos-3.9.yml index c2916485f39..c87767358b2 100644 --- a/conda-envs/mlos-3.9.yml +++ b/conda-envs/mlos-3.9.yml @@ -9,6 +9,7 @@ dependencies: - pycodestyle - autopep8 - pydocstyle + - flake8 - setuptools - setuptools-scm - jupyter diff --git a/conda-envs/mlos-windows.yml b/conda-envs/mlos-windows.yml index 8c561174445..663a1f686b0 100644 --- a/conda-envs/mlos-windows.yml +++ b/conda-envs/mlos-windows.yml @@ -9,6 +9,7 @@ dependencies: - pycodestyle - autopep8 - pydocstyle + - flake8 - setuptools - setuptools-scm - jupyter diff --git a/conda-envs/mlos.yml b/conda-envs/mlos.yml index 95caff41b72..6da3e88868f 100644 --- a/conda-envs/mlos.yml +++ b/conda-envs/mlos.yml @@ -9,6 +9,7 @@ dependencies: - pycodestyle - autopep8 - pydocstyle + - flake8 - setuptools - setuptools-scm - jupyter From f3988941f4be51369782b2f891bdfb468e0f4438 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Mon, 15 May 2023 21:21:28 +0000 Subject: [PATCH 06/12] accept some reformats --- .devcontainer/devcontainer.json | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 903903a33e6..6a8511891d7 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -8,7 +8,9 @@ "service": "app", "workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}", // Prep some host side things for the container build - "initializeCommand": [".devcontainer/scripts/prep-container-build"], + "initializeCommand": [ + ".devcontainer/scripts/prep-container-build" + ], // Make sure the container user can read/write to the package caches. "onCreateCommand": "sudo -n mkdir -p /opt/conda/pkgs/cache /var/cache/pip && sudo -n chown -R vscode /opt/conda/pkgs/cache /var/cache/pip", // Make sure the conda env is up to date with the source tree expectations. From 39cfef7002ebe0ba249b41915cd529127309ac11 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Mon, 15 May 2023 21:28:39 +0000 Subject: [PATCH 07/12] only format modifications --- .vscode/settings.json | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 31c9dc3d7e9..708ed30100b 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -10,12 +10,22 @@ "python.linting.pylintEnabled": true, "python.linting.pycodestyleEnabled": true, "python.linting.pydocstyleEnabled": true, - "python.linting.mypyEnabled": false, // use the dmypy daemon extension instead - it's faster + "python.linting.flake8Enabled": true, + "python.linting.mypyEnabled": false, // use the dmypy daemon extension instead - it's faster // "mypy.runUsingActiveInterpreter": true, // in the devcontainer we override this to use an explicit path "autoDocstring.docstringFormat": "numpy", "restructuredtext.linter.doc8.extraArgs": [ "--ignore D001" ], "esbonio.sphinx.confDir": "${workspaceFolder}/doc/source", - "esbonio.sphinx.buildDir": "${workspaceFolder}/doc/build/" + "esbonio.sphinx.buildDir": "${workspaceFolder}/doc/build/", + "editor.formatOnSave": true, + "editor.formatOnSaveMode": "modifications", + "editor.defaultFormatter": "esbenp.prettier-vscode", + "python.formatting.provider": "none", + "[python]": { + "editor.defaultFormatter": "ms-python.autopep8", + "editor.formatOnSave": true, + "editor.formatOnSaveMode": "modifications" + } } From 63648c52b52a10f0b8c90b6ecd9a9424bf6d5751 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Mon, 15 May 2023 21:28:52 +0000 Subject: [PATCH 08/12] some flake8 fixups --- mlos_bench/setup.py | 4 ++-- mlos_core/setup.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/mlos_bench/setup.py b/mlos_bench/setup.py index c8b65b6c167..3edbf4c2efe 100644 --- a/mlos_bench/setup.py +++ b/mlos_bench/setup.py @@ -18,14 +18,14 @@ from setuptools_scm import get_version version = get_version(root='..', relative_to=__file__) if version is not None: - _VERSION = version + _VERSION = version # noqa: F811 except ImportError: warning("setuptools_scm not found, using version from _version.py") except LookupError as e: warning(f"setuptools_scm failed to find git version, using version from _version.py: {e}") -extra_requires: Dict[str, List[str]] = { +extra_requires: Dict[str, List[str]] = { # pylint: disable=consider-using-namedtuple-or-dataclass # Additional tools for extra functionality. 'azure': ['azure-storage-file-share'], 'storage-sql-duckdb': ['sqlalchemy', 'duckdb_engine'], diff --git a/mlos_core/setup.py b/mlos_core/setup.py index 51d0e357d39..7ec91e2679f 100644 --- a/mlos_core/setup.py +++ b/mlos_core/setup.py @@ -18,14 +18,14 @@ from setuptools_scm import get_version version = get_version(root='..', relative_to=__file__) if version is not None: - _VERSION = version + _VERSION = version # noqa: F811 except ImportError: warning("setuptools_scm not found, using version from _version.py") except LookupError as e: warning(f"setuptools_scm failed to find git version, using version from _version.py: {e}") -extra_requires: Dict[str, List[str]] = { +extra_requires: Dict[str, List[str]] = { # pylint: disable=consider-using-namedtuple-or-dataclass 'emukit': ['emukit'], 'skopt': ['scikit-optimize<=0.9.0'], # FIXME: temporarily work around some version mismatch issues (PR 850) } @@ -52,10 +52,10 @@ '': ['py.typed', '**/*.pyi'], }, install_requires=[ - 'scikit-learn<1.2', # FIXME: temporarily work around some version mismatch issues (PR 850) - 'joblib>=1.1.1', # CVE-2022-21797: scikit-learn dependency, addressed in 1.2.0dev0, which isn't currently released + 'scikit-learn<1.2', # FIXME: temporarily work around some version mismatch issues (PR 850) + 'joblib>=1.1.1', # CVE-2022-21797: scikit-learn dependency, addressed in 1.2.0dev0, which isn't currently released 'scipy>=1.3.2', - 'numpy<1.24', # FIXME: temporarily work around some version mismatch issues (PR 850) + 'numpy<1.24', # FIXME: temporarily work around some version mismatch issues (PR 850) 'pandas>=1.0.3', 'ConfigSpace>=0.6.1', ], From d192f68e90b8b1e2501e9a33baf24c8f7ed1c0d1 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Mon, 15 May 2023 21:29:07 +0000 Subject: [PATCH 09/12] bring pycodestyle checks inline with flake8 --- setup.cfg | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.cfg b/setup.cfg index f6eb77d54aa..40991d05ac4 100644 --- a/setup.cfg +++ b/setup.cfg @@ -5,11 +5,10 @@ universal = 1 [pycodestyle] count = True # E124: Closing bracket does not match indentation of opening bracket's line -# E261: At least two spaces before inline comment # E502: The backslash is redundant between brackets # W503: Line break occurred before a binary operator # W504: Line break occurred after a binary operator -ignore = E124,E261,E502,W503,W504 +ignore = E124,E502,W503,W504 format = pylint # See Also: .editorconfig, .pylintrc max-line-length = 132 From b4a9e6a96950977884db72262a12fce4216ebd18 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Mon, 15 May 2023 21:29:31 +0000 Subject: [PATCH 10/12] add a prettier rc that instructs it to use our editorconfig --- .prettierrc | 4 ++++ 1 file changed, 4 insertions(+) create mode 100644 .prettierrc diff --git a/.prettierrc b/.prettierrc new file mode 100644 index 00000000000..cf0454734c8 --- /dev/null +++ b/.prettierrc @@ -0,0 +1,4 @@ +{ + "editorconfig": true, + "trailingComma": "es5" +} From 574bba216eae2d169b7c95119ccc63bff1308ad8 Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Mon, 15 May 2023 21:48:18 +0000 Subject: [PATCH 11/12] PR comments --- .pylintrc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pylintrc b/.pylintrc index bb0ab42cd0d..d373c159b58 100644 --- a/.pylintrc +++ b/.pylintrc @@ -8,7 +8,7 @@ fail-under=9.8 # See Also: https://github.com/PyCQA/pydocstyle/issues/309#issuecomment-1426642147 # Also fail on unused imports. fail-on= - C0116, + missing-function-docstring, unused-import # Ignore pylint complaints about an upstream dependency. From 1cec61b891785b3f6e3012a56eaeb6879211002f Mon Sep 17 00:00:00 2001 From: Brian Kroth Date: Mon, 15 May 2023 21:48:34 +0000 Subject: [PATCH 12/12] PR comments --- setup.cfg | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/setup.cfg b/setup.cfg index 40991d05ac4..78b87728ed5 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,11 +4,9 @@ universal = 1 [pycodestyle] count = True -# E124: Closing bracket does not match indentation of opening bracket's line -# E502: The backslash is redundant between brackets # W503: Line break occurred before a binary operator # W504: Line break occurred after a binary operator -ignore = E124,E502,W503,W504 +ignore = W503,W504 format = pylint # See Also: .editorconfig, .pylintrc max-line-length = 132