diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 8525435..3fdf799 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -43,20 +43,10 @@ "python.testing.unittestEnabled": false, "python.testing.pytestArgs": ["."], "python.analysis.typeCheckingMode": "off", - "python.linting.enabled": true, - "python.linting.flake8Enabled": false, - "python.formatting.blackPath": "/usr/local/py-utils/bin/black", - "python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle", - "python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle", - "python.linting.mypyPath": "/usr/local/py-utils/bin/mypy", - "python.linting.pylintPath": "/usr/local/py-utils/bin/pylint", - "http.systemCertificates": true, - "python.formatting.provider": "black", "black-formatter.args": ["--preview"], "files.associations": { "*.json": "jsonc" }, - "python.linting.pylintEnabled": true, "files.autoSave": "off", // needed for editor.formatOnSave "files.insertFinalNewline": true, // needed for pre-commit check "files.trimTrailingWhitespace": true, @@ -69,8 +59,12 @@ }, "python.testing.pytestEnabled": true, "editor.formatOnType": true, - "ruff.run": "onType", - "mypy.enabled": true + "ruff.lint.enable": true, + "ruff.lint.run": "onType", + "mypy.enabled": true, + "[python]": { + "editor.defaultFormatter": "ms-python.black-formatter" + } } } } diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 3dadd35..a24af84 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -109,7 +109,7 @@ jobs: - name: Run ruff run: | . venv/bin/activate - pre-commit run --hook-stage manual ruff --all-files --show-diff-on-failure + pre-commit run ruff --all-files --show-diff-on-failure lint-black: name: black @@ -144,7 +144,7 @@ jobs: - name: Run black checks run: | . venv/bin/activate - pre-commit run --hook-stage manual black --all-files --show-diff-on-failure + pre-commit run black --all-files --show-diff-on-failure pylint: name: Check pylint diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ad00f0a..0c2b530 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -13,7 +13,6 @@ repos: - id: check-yaml - id: check-added-large-files - id: check-executables-have-shebangs - stages: [manual] - id: no-commit-to-branch args: - --branch=dev @@ -36,7 +35,6 @@ repos: rev: 24.1.1 hooks: - id: black - stages: [manual] args: - --check - repo: https://github.com/codespell-project/codespell diff --git a/.vscode/settings.json b/.vscode/settings.json index 0fd764e..851f72b 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -1,11 +1,10 @@ { "python.testing.unittestArgs": ["-v", "-s", "tests", "-p", "test_*.py"], + "python.defaultInterpreterPath": "/usr/local/bin/python", "python.testing.unittestEnabled": false, + "python.testing.pytestArgs": ["."], "python.analysis.typeCheckingMode": "off", - "python.linting.enabled": true, - "python.linting.flake8Enabled": false, - "http.systemCertificates": true, - "python.formatting.provider": "black", + "black-formatter.args": ["--preview"], "files.associations": { "*.json": "jsonc" }, @@ -17,15 +16,14 @@ "isort.args": ["--profile", "black"], "pylint.severity": { "convention": "Warning", - "refactor": "Warning", "info": "Warning" }, - "black-formatter.args": ["--preview"], - "mypy.enabled": true, - "ruff.run": "onType", "python.testing.pytestEnabled": true, - "python.linting.pylintEnabled": true, "editor.formatOnType": true, - "python.testing.pytestArgs": [".", "--keepalive"], - "python.linting.pylintCategorySeverity.refactor": "Warning" + "ruff.lint.enable": true, + "ruff.lint.run": "onType", + "mypy.enabled": true, + "[python]": { + "editor.defaultFormatter": "ms-python.black-formatter" + } } diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ee865c..7e0bce3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## v24.14.2 + +- `black` formatting fix + ## v24.14.1 - Add test coverage for s3 file renames when using proxy transfer type diff --git a/pyproject.toml b/pyproject.toml index b208f2e..bf1ae9e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "otf-addons-aws" -version = "v24.14.1" +version = "v24.14.2" authors = [{ name = "Adam McDonagh", email = "adam@elitemonkey.net" }] license = { text = "GPLv3" } classifiers = [ @@ -52,7 +52,7 @@ dev = [ profile = 'black' [tool.bumpver] -current_version = "v24.14.1" +current_version = "v24.14.2" version_pattern = "vYY.WW.PATCH[-TAG]" commit_message = "bump version {old_version} -> {new_version}" commit = true diff --git a/src/opentaskpy/addons/aws/remotehandlers/s3.py b/src/opentaskpy/addons/aws/remotehandlers/s3.py index 221b44d..4520876 100644 --- a/src/opentaskpy/addons/aws/remotehandlers/s3.py +++ b/src/opentaskpy/addons/aws/remotehandlers/s3.py @@ -342,9 +342,7 @@ def push_files_from_worker( rename_sub = self.spec["rename"]["sub"] file_name = re.sub(rename_regex, rename_sub, file_name) - self.logger.info( - f"Renaming file to {file_name}" - ) + self.logger.info(f"Renaming file to {file_name}") self.logger.info( f"Transferring file: {file} to" f" s3://{self.spec['bucket']}/{self.spec['directory']}/{file_name}" @@ -433,9 +431,7 @@ def transfer_files( rename_sub = dest_remote_handler.spec["rename"]["sub"] file_name = re.sub(rename_regex, rename_sub, file_name) - self.logger.info( - f"Renaming file to {file_name}" - ) + self.logger.info(f"Renaming file to {file_name}") self.logger.info( f"Transferring file: {file} from {self.spec['bucket']} to" f" {dest_remote_handler.spec['bucket']}/{file_name}" diff --git a/tests/test_remotehandler_s3_transfer.py b/tests/test_remotehandler_s3_transfer.py index c72ea01..a49ac4f 100644 --- a/tests/test_remotehandler_s3_transfer.py +++ b/tests/test_remotehandler_s3_transfer.py @@ -322,10 +322,7 @@ { "bucket": BUCKET_NAME_2, "directory": "dest", - "rename" : { - "pattern": "abc", - "sub" : "def" - }, + "rename": {"pattern": "abc", "sub": "def"}, "protocol": { "name": "opentaskpy.addons.aws.remotehandlers.s3.S3Transfer", }, @@ -347,10 +344,7 @@ { "bucket": BUCKET_NAME_2, "directory": "dest", - "rename" : { - "pattern": "abc", - "sub" : "def" - }, + "rename": {"pattern": "abc", "sub": "def"}, "transferType": "proxy", "protocol": { "name": "opentaskpy.addons.aws.remotehandlers.s3.S3Transfer", @@ -850,6 +844,7 @@ def test_s3_to_s3_copy_pca_rename(setup_bucket, tmp_path, s3_client): objects["Contents"][0]["Key"] == "src/archive/file-pca-rename-1234-renamed.txt" ) + def test_s3_to_s3_copy_rename(setup_bucket, tmp_path, s3_client): transfer_obj = transfer.Transfer( None, "s3-to-s3-rename", s3_to_s3_rename_task_definition @@ -870,13 +865,16 @@ def test_s3_to_s3_copy_rename(setup_bucket, tmp_path, s3_client): assert len(objects["Contents"]) == 1 assert objects["Contents"][0]["Key"] == "dest/file-rename-def.txt" + def test_s3_to_s3_proxy_rename(setup_bucket, s3_client, tmp_path): transfer_obj = transfer.Transfer( None, "s3-to-s3-proxy-rename", s3_to_s3_proxy_rename_task_definition ) # Write a test file locally - fs.create_files([{f"{tmp_path}/file-rename-proxy-abc.txt": {"content": "test1234"}}]) + fs.create_files( + [{f"{tmp_path}/file-rename-proxy-abc.txt": {"content": "test1234"}}] + ) create_s3_file( s3_client, f"{tmp_path}/file-rename-proxy-abc.txt", @@ -890,6 +888,7 @@ def test_s3_to_s3_proxy_rename(setup_bucket, s3_client, tmp_path): assert len(objects["Contents"]) == 1 assert objects["Contents"][0]["Key"] == "dest/file-rename-proxy-def.txt" + def test_s3_file_watch_custom_creds( setup_bucket, tmp_path,