From 828507ebb8e4f74ec6eb08b181bf994534f38b5c Mon Sep 17 00:00:00 2001 From: Kasper Date: Wed, 6 Oct 2021 04:44:20 +0200 Subject: [PATCH 01/14] Exclude .venv from Time Machine backups --- poetry.lock | 21 ++++++++++++++++++++- poetry/utils/env.py | 16 +++++++++++++++- pyproject.toml | 1 + tests/utils/test_env.py | 15 +++++++++++++++ 4 files changed, 51 insertions(+), 2 deletions(-) diff --git a/poetry.lock b/poetry.lock index 02dc2635f97..6ef12e05762 100644 --- a/poetry.lock +++ b/poetry.lock @@ -688,6 +688,17 @@ category = "main" optional = false python-versions = "*" +[[package]] +name = "xattr" +version = "0.9.7" +description = "Python wrapper for extended filesystem attributes" +category = "main" +optional = false +python-versions = "*" + +[package.dependencies] +cffi = ">=1.0.0" + [[package]] name = "zipp" version = "3.5.0" @@ -703,7 +714,7 @@ testing = ["pytest (>=4.6)", "pytest-checkdocs (>=2.4)", "pytest-flake8", "pytes [metadata] lock-version = "1.1" python-versions = "^3.6" -content-hash = "458035eaa19225869083c98c699e753de1844dd82030447634836e2f79255df1" +content-hash = "cd1b5d281fcb078f3734d3d1332fde916c42b6ae43f6ccc2d3b4e57566019bd9" [metadata.files] appdirs = [ @@ -861,8 +872,10 @@ cryptography = [ {file = "cryptography-3.4.7-cp36-abi3-win_amd64.whl", hash = "sha256:de4e5f7f68220d92b7637fc99847475b59154b7a1b3868fb7385337af54ac9ca"}, {file = "cryptography-3.4.7-pp36-pypy36_pp73-manylinux2010_x86_64.whl", hash = "sha256:26965837447f9c82f1855e0bc8bc4fb910240b6e0d16a664bb722df3b5b06873"}, {file = "cryptography-3.4.7-pp36-pypy36_pp73-manylinux2014_x86_64.whl", hash = "sha256:eb8cc2afe8b05acbd84a43905832ec78e7b3873fb124ca190f574dca7389a87d"}, + {file = "cryptography-3.4.7-pp37-pypy37_pp73-macosx_10_10_x86_64.whl", hash = "sha256:b01fd6f2737816cb1e08ed4807ae194404790eac7ad030b34f2ce72b332f5586"}, {file = "cryptography-3.4.7-pp37-pypy37_pp73-manylinux2010_x86_64.whl", hash = "sha256:7ec5d3b029f5fa2b179325908b9cd93db28ab7b85bb6c1db56b10e0b54235177"}, {file = "cryptography-3.4.7-pp37-pypy37_pp73-manylinux2014_x86_64.whl", hash = "sha256:ee77aa129f481be46f8d92a1a7db57269a2f23052d5f2433b4621bb457081cc9"}, + {file = "cryptography-3.4.7-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:bf40af59ca2465b24e54f671b2de2c59257ddc4f7e5706dbd6930e26823668d3"}, {file = "cryptography-3.4.7.tar.gz", hash = "sha256:3d10de8116d25649631977cb37da6cbdd2d6fa0e0281d014a5b7d337255ca713"}, ] dataclasses = [ @@ -1102,6 +1115,12 @@ webencodings = [ {file = "webencodings-0.5.1-py2.py3-none-any.whl", hash = "sha256:a0af1213f3c2226497a97e2b3aa01a7e4bee4f403f95be16fc9acd2947514a78"}, {file = "webencodings-0.5.1.tar.gz", hash = "sha256:b36a1c245f2d304965eb4e0a82848379241dc04b865afcc4aab16748587e1923"}, ] +xattr = [ + {file = "xattr-0.9.7-cp27-cp27m-macosx_10_13_x86_64.whl", hash = "sha256:1b2cd125150aa9bbfb02929627101b3303920a68487e9c865ddd170188ddd796"}, + {file = "xattr-0.9.7-cp36-cp36m-macosx_10_13_x86_64.whl", hash = "sha256:e2c72a3a501bac715489180ca2b646e48a1ca3a794c1103dd6f0f987d43f570c"}, + {file = "xattr-0.9.7-cp37-cp37m-macosx_10_13_x86_64.whl", hash = "sha256:1e11ba8ab86dfe74419704c53722ea9b5915833db07416e7c10db5dfb02218bb"}, + {file = "xattr-0.9.7.tar.gz", hash = "sha256:b0bbca828e04ef2d484a6522ae7b3a7ccad5e43fa1c6f54d78e24bb870f49d44"}, +] zipp = [ {file = "zipp-3.5.0-py3-none-any.whl", hash = "sha256:957cfda87797e389580cb8b9e3870841ca991e2125350677b2ca83a0e99390a3"}, {file = "zipp-3.5.0.tar.gz", hash = "sha256:f5812b1e007e48cff63449a5e9f4e7ebea716b4111f9c4f9a645f91d579bf0c4"}, diff --git a/poetry/utils/env.py b/poetry/utils/env.py index 4a6f42291ac..14f1bb4403c 100644 --- a/poetry/utils/env.py +++ b/poetry/utils/env.py @@ -28,6 +28,7 @@ import packaging.tags import tomlkit import virtualenv +import xattr from cleo.io.io import IO from packaging.tags import Tag @@ -1002,7 +1003,20 @@ def build_venv( args.append(str(path)) - return virtualenv.cli_run(args) + cli_result = virtualenv.cli_run(args) + + print("path", str(path)) + print("args", args) + + # Exclude the venv folder from from macOS Time Machine backups + if sys.platform == "darwin": + xattr.setxattr( + str(path), + "com.apple.metadata:com_apple_backup_excludeItem", + b"bplist00_\x10\x11com.apple.backupd\x08\x00\x00\x00\x00\x00\x00\x01\x01\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1c", + ) + + return cli_result @classmethod def remove_venv(cls, path: Union[Path, str]) -> None: diff --git a/pyproject.toml b/pyproject.toml index a049e1f8369..45cae103f39 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,6 +50,7 @@ keyring = ">=21.2.0" entrypoints = "^0.3" importlib-metadata = {version = "^1.6.0", python = "<3.8"} dataclasses = {version = "^0.8", python = "~3.6"} +xattr = "^0.9.7" [tool.poetry.dev-dependencies] pytest = "^6.2" diff --git a/tests/utils/test_env.py b/tests/utils/test_env.py index d7535553758..cb68833bd9a 100644 --- a/tests/utils/test_env.py +++ b/tests/utils/test_env.py @@ -9,6 +9,7 @@ import pytest import tomlkit +import xattr from cleo.io.null_io import NullIO @@ -77,6 +78,20 @@ def test_virtualenvs_with_spaces_in_their_path_work_as_expected(tmp_dir, manager assert venv.run("python", "-V", shell=True).startswith("Python") +def test_venv_backup_exclusion(tmp_dir, manager): + venv_path = Path(tmp_dir) / "Virtual Env" + + manager.build_venv(str(venv_path)) + + if sys.platform == "darwin": + assert ( + xattr.getxattr( + str(venv_path), "com.apple.metadata:com_apple_backup_excludeItem" + ) + == b"bplist00_\x10\x11com.apple.backupd\x08\x00\x00\x00\x00\x00\x00\x01\x01\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1c" + ) + + def test_env_commands_with_spaces_in_their_arg_work_as_expected(tmp_dir, manager): venv_path = Path(tmp_dir) / "Virtual Env" manager.build_venv(str(venv_path)) From e0a4fd9c7908aa88095dc2fb9cd5665011178ddb Mon Sep 17 00:00:00 2001 From: Kasper Date: Wed, 6 Oct 2021 17:05:40 +0200 Subject: [PATCH 02/14] Remove leftover print statements --- poetry/utils/env.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/poetry/utils/env.py b/poetry/utils/env.py index 14f1bb4403c..a2c34b9849a 100644 --- a/poetry/utils/env.py +++ b/poetry/utils/env.py @@ -1005,9 +1005,6 @@ def build_venv( cli_result = virtualenv.cli_run(args) - print("path", str(path)) - print("args", args) - # Exclude the venv folder from from macOS Time Machine backups if sys.platform == "darwin": xattr.setxattr( From 345caf31c59194fd6eb06fa622ddbc4a02d377ce Mon Sep 17 00:00:00 2001 From: Kasper Date: Wed, 6 Oct 2021 17:07:05 +0200 Subject: [PATCH 03/14] Make backup exclusion test only run on darwin --- tests/utils/test_env.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/utils/test_env.py b/tests/utils/test_env.py index cb68833bd9a..8fbb2fcc80a 100644 --- a/tests/utils/test_env.py +++ b/tests/utils/test_env.py @@ -78,18 +78,18 @@ def test_virtualenvs_with_spaces_in_their_path_work_as_expected(tmp_dir, manager assert venv.run("python", "-V", shell=True).startswith("Python") +@pytest.mark.skipif(sys.platform != "darwin", reason="requires darwin") def test_venv_backup_exclusion(tmp_dir, manager): venv_path = Path(tmp_dir) / "Virtual Env" manager.build_venv(str(venv_path)) - if sys.platform == "darwin": - assert ( - xattr.getxattr( - str(venv_path), "com.apple.metadata:com_apple_backup_excludeItem" - ) - == b"bplist00_\x10\x11com.apple.backupd\x08\x00\x00\x00\x00\x00\x00\x01\x01\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1c" + assert ( + xattr.getxattr( + str(venv_path), "com.apple.metadata:com_apple_backup_excludeItem" ) + == b"bplist00_\x10\x11com.apple.backupd\x08\x00\x00\x00\x00\x00\x00\x01\x01\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1c" + ) def test_env_commands_with_spaces_in_their_arg_work_as_expected(tmp_dir, manager): From ba472661d703e9d13b12c1af04507f9f1cba1824 Mon Sep 17 00:00:00 2001 From: Kasper Date: Wed, 6 Oct 2021 17:51:55 +0200 Subject: [PATCH 04/14] Add comment explaining backup exclusion xattr value --- poetry/utils/env.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/poetry/utils/env.py b/poetry/utils/env.py index a2c34b9849a..02d66cf8034 100644 --- a/poetry/utils/env.py +++ b/poetry/utils/env.py @@ -1010,6 +1010,8 @@ def build_venv( xattr.setxattr( str(path), "com.apple.metadata:com_apple_backup_excludeItem", + # This is the value `tmutil addexclusion ` sets. + # It's a binary plist encoding of the string "com.apple.backupd". b"bplist00_\x10\x11com.apple.backupd\x08\x00\x00\x00\x00\x00\x00\x01\x01\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1c", ) From 3cb4da059ea570452120ddaf4ae75631f65d7439 Mon Sep 17 00:00:00 2001 From: Kasper Date: Wed, 6 Oct 2021 18:41:43 +0200 Subject: [PATCH 05/14] Use plistlib to generate backup exclusion xattr value --- poetry/utils/env.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/poetry/utils/env.py b/poetry/utils/env.py index 02d66cf8034..7d6292cb309 100644 --- a/poetry/utils/env.py +++ b/poetry/utils/env.py @@ -4,6 +4,7 @@ import json import os import platform +import plistlib import re import shutil import subprocess @@ -1010,9 +1011,7 @@ def build_venv( xattr.setxattr( str(path), "com.apple.metadata:com_apple_backup_excludeItem", - # This is the value `tmutil addexclusion ` sets. - # It's a binary plist encoding of the string "com.apple.backupd". - b"bplist00_\x10\x11com.apple.backupd\x08\x00\x00\x00\x00\x00\x00\x01\x01\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1c", + plistlib.dumps("com.apple.backupd", fmt=plistlib.FMT_BINARY), ) return cli_result From 255b90d5efa36a1c232bdd1344f4763b58c9df81 Mon Sep 17 00:00:00 2001 From: Kasper Date: Thu, 7 Oct 2021 20:16:22 +0200 Subject: [PATCH 06/14] Only install xattr on darwin --- poetry/utils/env.py | 3 ++- pyproject.toml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/poetry/utils/env.py b/poetry/utils/env.py index 7d6292cb309..2a3aa5a635b 100644 --- a/poetry/utils/env.py +++ b/poetry/utils/env.py @@ -29,7 +29,6 @@ import packaging.tags import tomlkit import virtualenv -import xattr from cleo.io.io import IO from packaging.tags import Tag @@ -1008,6 +1007,8 @@ def build_venv( # Exclude the venv folder from from macOS Time Machine backups if sys.platform == "darwin": + import xattr + xattr.setxattr( str(path), "com.apple.metadata:com_apple_backup_excludeItem", diff --git a/pyproject.toml b/pyproject.toml index 45cae103f39..04e060a0a22 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,7 +50,7 @@ keyring = ">=21.2.0" entrypoints = "^0.3" importlib-metadata = {version = "^1.6.0", python = "<3.8"} dataclasses = {version = "^0.8", python = "~3.6"} -xattr = "^0.9.7" +xattr = {version = "^0.9.7", markers = "sys_platform == 'darwin'"} [tool.poetry.dev-dependencies] pytest = "^6.2" From 17c625d80de8f5358a42d7ff20d862db7eb30ce8 Mon Sep 17 00:00:00 2001 From: Kasper Date: Sun, 5 Jun 2022 01:59:14 +0200 Subject: [PATCH 07/14] Add TODO comment --- poetry/utils/env.py | 1 + 1 file changed, 1 insertion(+) diff --git a/poetry/utils/env.py b/poetry/utils/env.py index 2a3aa5a635b..b50a6b513bf 100644 --- a/poetry/utils/env.py +++ b/poetry/utils/env.py @@ -1006,6 +1006,7 @@ def build_venv( cli_result = virtualenv.cli_run(args) # Exclude the venv folder from from macOS Time Machine backups + # TODO: Add backup-ignore markers for other platforms too if sys.platform == "darwin": import xattr From 8878f30f2e322d9762902af02a8d9a3c7d7a86cd Mon Sep 17 00:00:00 2001 From: Kasper Date: Sun, 5 Jun 2022 02:11:25 +0200 Subject: [PATCH 08/14] Merge fix --- tests/utils/test_env.py | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/tests/utils/test_env.py b/tests/utils/test_env.py index a656e2f2d99..5ddd70ffa1d 100644 --- a/tests/utils/test_env.py +++ b/tests/utils/test_env.py @@ -110,16 +110,6 @@ def test_venv_backup_exclusion(tmp_dir, manager): == b"bplist00_\x10\x11com.apple.backupd\x08\x00\x00\x00\x00\x00\x00\x01\x01\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1c" ) - -def test_env_commands_with_spaces_in_their_arg_work_as_expected(tmp_dir, manager): - venv_path = Path(tmp_dir) / "Virtual Env" - manager.build_venv(str(venv_path)) - venv = VirtualEnv(venv_path) - assert venv.run("python", venv.pip, "--version", shell=True).startswith( - "pip {} from ".format(venv.pip_version) - ) - - def test_env_commands_with_spaces_in_their_arg_work_as_expected( tmp_dir: str, manager: EnvManager ): From 8b062c0ba9bfb5087bf255cdcc5f6b68c7f104af Mon Sep 17 00:00:00 2001 From: Kasper Date: Sun, 5 Jun 2022 02:20:05 +0200 Subject: [PATCH 09/14] Only import xattr on macOS test --- tests/utils/test_env.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/utils/test_env.py b/tests/utils/test_env.py index 5ddd70ffa1d..69960837f64 100644 --- a/tests/utils/test_env.py +++ b/tests/utils/test_env.py @@ -10,7 +10,8 @@ import pytest import tomlkit -import xattr +if sys.platform == "darwin": + import xattr from cleo.io.null_io import NullIO from poetry.core.semver.version import Version From 55c854d90517737772955d8fc6aa17289e61b2a3 Mon Sep 17 00:00:00 2001 From: Kasper Date: Sun, 5 Jun 2022 02:28:21 +0200 Subject: [PATCH 10/14] Try fixing CI error --- tests/utils/test_env.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/utils/test_env.py b/tests/utils/test_env.py index 69960837f64..273f0ceb63c 100644 --- a/tests/utils/test_env.py +++ b/tests/utils/test_env.py @@ -10,8 +10,6 @@ import pytest import tomlkit -if sys.platform == "darwin": - import xattr from cleo.io.null_io import NullIO from poetry.core.semver.version import Version @@ -100,6 +98,8 @@ def test_virtualenvs_with_spaces_in_their_path_work_as_expected( @pytest.mark.skipif(sys.platform != "darwin", reason="requires darwin") def test_venv_backup_exclusion(tmp_dir, manager): + import xattr + venv_path = Path(tmp_dir) / "Virtual Env" manager.build_venv(str(venv_path)) From b274f22771eca521d9795cbc0b1a5b07695e3666 Mon Sep 17 00:00:00 2001 From: Kasper Date: Sun, 5 Jun 2022 02:30:37 +0200 Subject: [PATCH 11/14] Add type annotations --- tests/utils/test_env.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/utils/test_env.py b/tests/utils/test_env.py index 273f0ceb63c..8d34d87ed57 100644 --- a/tests/utils/test_env.py +++ b/tests/utils/test_env.py @@ -97,7 +97,7 @@ def test_virtualenvs_with_spaces_in_their_path_work_as_expected( @pytest.mark.skipif(sys.platform != "darwin", reason="requires darwin") -def test_venv_backup_exclusion(tmp_dir, manager): +def test_venv_backup_exclusion(tmp_dir: str, manager: EnvManager): import xattr venv_path = Path(tmp_dir) / "Virtual Env" From 099ec09bf8bde1a69342036cd82bde25cc2fdbb3 Mon Sep 17 00:00:00 2001 From: Kasper Date: Sun, 5 Jun 2022 02:47:47 +0200 Subject: [PATCH 12/14] Fix CI errors --- src/poetry/utils/env.py | 2 +- tests/utils/test_env.py | 8 +++++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/poetry/utils/env.py b/src/poetry/utils/env.py index b671c37253c..11be5bb1da7 100644 --- a/src/poetry/utils/env.py +++ b/src/poetry/utils/env.py @@ -1093,7 +1093,7 @@ def build_venv( # Exclude the venv folder from from macOS Time Machine backups # TODO: Add backup-ignore markers for other platforms too if sys.platform == "darwin": - import xattr + import xattr # type: ignore[import] xattr.setxattr( str(path), diff --git a/tests/utils/test_env.py b/tests/utils/test_env.py index 8d34d87ed57..ecb0fdc6f80 100644 --- a/tests/utils/test_env.py +++ b/tests/utils/test_env.py @@ -104,13 +104,19 @@ def test_venv_backup_exclusion(tmp_dir: str, manager: EnvManager): manager.build_venv(str(venv_path)) + value = ( + b"bplist00_\x10\x11com.apple.backupd" + b"\x08\x00\x00\x00\x00\x00\x00\x01\x01\x00\x00\x00\x00\x00\x00\x00" + b"\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1c" + ) assert ( xattr.getxattr( str(venv_path), "com.apple.metadata:com_apple_backup_excludeItem" ) - == b"bplist00_\x10\x11com.apple.backupd\x08\x00\x00\x00\x00\x00\x00\x01\x01\x00\x00\x00\x00\x00\x00\x00\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x1c" + == value ) + def test_env_commands_with_spaces_in_their_arg_work_as_expected( tmp_dir: str, manager: EnvManager ): From 48b4ea0fd78bf146c4c1061a9cf8a0066a77fe0f Mon Sep 17 00:00:00 2001 From: Kasper Date: Sun, 5 Jun 2022 08:50:48 +0200 Subject: [PATCH 13/14] Add spacing pyproject.toml Co-authored-by: Bjorn Neergaard --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 209b931cb6d..c954b8fce33 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -66,7 +66,7 @@ shellingham = "^1.1" tomlkit = ">=0.7.0,<1.0.0" # exclude 20.4.5 - 20.4.6 due to https://github.com/pypa/pip/issues/9953 virtualenv = "(>=20.4.3,<20.4.5 || >=20.4.7)" -xattr = {version = "^0.9.7", markers = "sys_platform == 'darwin'"} +xattr = { version = "^0.9.7", markers = "sys_platform == 'darwin'" } urllib3 = "^1.26.0" dulwich = "^0.20.35" From 7dcc9ec6d54ff9236fd271618bcc8ffbff1dd5a4 Mon Sep 17 00:00:00 2001 From: Kasper Date: Sun, 5 Jun 2022 08:53:27 +0200 Subject: [PATCH 14/14] Move mypy ignore to `pyproject.toml` --- pyproject.toml | 1 + src/poetry/utils/env.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 209b931cb6d..04a01834525 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -158,6 +158,7 @@ module = [ 'requests_toolbelt.*', 'shellingham.*', 'virtualenv.*', + 'xattr.*', ] ignore_missing_imports = true diff --git a/src/poetry/utils/env.py b/src/poetry/utils/env.py index 11be5bb1da7..b671c37253c 100644 --- a/src/poetry/utils/env.py +++ b/src/poetry/utils/env.py @@ -1093,7 +1093,7 @@ def build_venv( # Exclude the venv folder from from macOS Time Machine backups # TODO: Add backup-ignore markers for other platforms too if sys.platform == "darwin": - import xattr # type: ignore[import] + import xattr xattr.setxattr( str(path),