From bc9b3e7242cebb85882e601134750d11b6e9b206 Mon Sep 17 00:00:00 2001 From: justindujardin Date: Mon, 1 Feb 2021 15:06:20 -0800 Subject: [PATCH 1/4] feat(tests): include tests in pypi package - requested to support testing new versions of pathy recipes - allow overriding test buckets for GCS client testing --- MANIFEST.in | 1 + {tests => pathy/tests}/__init__.py | 0 {tests => pathy/tests}/conftest.py | 12 ++++++------ .../tests}/fixtures/tar_but_not_gzipped.tar.gz | Bin {tests => pathy/tests}/test_base.py | 0 {tests => pathy/tests}/test_cli.py | 0 {tests => pathy/tests}/test_clients.py | 0 {tests => pathy/tests}/test_file.py | 0 tools/test.sh | 2 +- 9 files changed, 8 insertions(+), 7 deletions(-) rename {tests => pathy/tests}/__init__.py (100%) rename {tests => pathy/tests}/conftest.py (91%) rename {tests => pathy/tests}/fixtures/tar_but_not_gzipped.tar.gz (100%) rename {tests => pathy/tests}/test_base.py (100%) rename {tests => pathy/tests}/test_cli.py (100%) rename {tests => pathy/tests}/test_clients.py (100%) rename {tests => pathy/tests}/test_file.py (100%) diff --git a/MANIFEST.in b/MANIFEST.in index edbe785..ec46205 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,3 +3,4 @@ include README.md include LICENSE include pathy/py.typed include requirements.txt +include pathy/tests/fixtures/* diff --git a/tests/__init__.py b/pathy/tests/__init__.py similarity index 100% rename from tests/__init__.py rename to pathy/tests/__init__.py diff --git a/tests/conftest.py b/pathy/tests/conftest.py similarity index 91% rename from tests/conftest.py rename to pathy/tests/conftest.py index 9e81e27..13813a5 100644 --- a/tests/conftest.py +++ b/pathy/tests/conftest.py @@ -18,12 +18,12 @@ @pytest.fixture() def bucket() -> str: - return "pathy-tests-bucket" + return os.environ.get("PATHY_TEST_BUCKET", "pathy-tests-bucket") @pytest.fixture() def other_bucket() -> str: - return "pathy-tests-bucket-other" + return os.environ.get("PATHY_TEST_BUCKET_OTHER", "pathy-tests-bucket-other") @pytest.fixture() @@ -62,10 +62,7 @@ def credentials_from_env(): except json.decoder.JSONDecodeError: pass - # If not a file path, assume it's JSON content - if json_creds is None: - credentials = service_account.Credentials.from_service_account_file(creds) - else: + if json_creds is not None: fd, path = tempfile.mkstemp() try: with os.fdopen(fd, "w") as tmp: @@ -73,6 +70,9 @@ def credentials_from_env(): credentials = service_account.Credentials.from_service_account_file(path) finally: os.remove(path) + else: + # If not a JSON string, assume it's a JSON file path + credentials = service_account.Credentials.from_service_account_file(creds) return credentials diff --git a/tests/fixtures/tar_but_not_gzipped.tar.gz b/pathy/tests/fixtures/tar_but_not_gzipped.tar.gz similarity index 100% rename from tests/fixtures/tar_but_not_gzipped.tar.gz rename to pathy/tests/fixtures/tar_but_not_gzipped.tar.gz diff --git a/tests/test_base.py b/pathy/tests/test_base.py similarity index 100% rename from tests/test_base.py rename to pathy/tests/test_base.py diff --git a/tests/test_cli.py b/pathy/tests/test_cli.py similarity index 100% rename from tests/test_cli.py rename to pathy/tests/test_cli.py diff --git a/tests/test_clients.py b/pathy/tests/test_clients.py similarity index 100% rename from tests/test_clients.py rename to pathy/tests/test_clients.py diff --git a/tests/test_file.py b/pathy/tests/test_file.py similarity index 100% rename from tests/test_file.py rename to pathy/tests/test_file.py diff --git a/tools/test.sh b/tools/test.sh index 140a7b1..2c81278 100644 --- a/tools/test.sh +++ b/tools/test.sh @@ -3,5 +3,5 @@ set -e echo "Activating virtualenv... (if this fails you may need to run setup.sh first)" . .env/bin/activate echo "Running tests..." -pytest tests --cov=pathy +pytest pathy/tests --cov=pathy From 3774ad0efbd2227aaa5b35b56b5bf774424fbc79 Mon Sep 17 00:00:00 2001 From: justindujardin Date: Mon, 1 Feb 2021 15:31:32 -0800 Subject: [PATCH 2/4] chore: fix lint in tests --- pathy/tests/conftest.py | 13 ++- pathy/tests/test_base.py | 187 +++++++++++++++++++----------------- pathy/tests/test_cli.py | 24 +++-- pathy/tests/test_clients.py | 21 ++-- pathy/tests/test_file.py | 8 +- tools/format.sh | 8 +- tools/lint.sh | 6 +- 7 files changed, 141 insertions(+), 126 deletions(-) diff --git a/pathy/tests/conftest.py b/pathy/tests/conftest.py index 13813a5..b2718be 100644 --- a/pathy/tests/conftest.py +++ b/pathy/tests/conftest.py @@ -3,6 +3,7 @@ import shutil import tempfile from pathlib import Path +from typing import Any, Generator, Optional import pytest @@ -27,20 +28,20 @@ def other_bucket() -> str: @pytest.fixture() -def temp_folder(): +def temp_folder() -> Generator[Path, None, None]: tmp_dir = tempfile.mkdtemp() yield Path(tmp_dir) shutil.rmtree(tmp_dir) @pytest.fixture() -def with_fs(temp_folder): +def with_fs(temp_folder: Path) -> Generator[Path, None, None]: yield temp_folder # Turn off FS adapter use_fs(False) -def credentials_from_env(): +def gcs_credentials_from_env() -> Optional[Any]: """Extract a credentials instance from the GCS_CREDENTIALS env variable. You can specify the contents of a credentials JSON file or a file path @@ -77,13 +78,15 @@ def credentials_from_env(): @pytest.fixture() -def with_adapter(adapter: str, bucket: str, other_bucket: str): +def with_adapter( + adapter: str, bucket: str, other_bucket: str +) -> Generator[str, None, None]: tmp_dir = None scheme = "gs" if adapter == "gcs": # Use GCS use_fs(False) - credentials = credentials_from_env() + credentials = gcs_credentials_from_env() if credentials is not None: set_client_params("gs", credentials=credentials) elif adapter == "fs": diff --git a/pathy/tests/test_base.py b/pathy/tests/test_base.py index 17002fa..7154252 100644 --- a/pathy/tests/test_base.py +++ b/pathy/tests/test_base.py @@ -1,6 +1,7 @@ import os import sys from pathlib import Path, PurePosixPath, PureWindowsPath +from typing import Any from uuid import uuid4 import pytest @@ -29,91 +30,91 @@ # todo(jd): replace global test-bucket with mock or generate buckets and call these e2e tests -def test_base_package_declares_version(): +def test_base_package_declares_version() -> None: assert __version__ is not None -def test_base_not_supported(monkeypatch): +def test_base_not_supported(monkeypatch: Any) -> None: monkeypatch.setattr(Pathy._flavour, "is_supported", False) with pytest.raises(NotImplementedError): Pathy() -def test_base_cwd(): +def test_base_cwd() -> None: with pytest.raises(NotImplementedError): Pathy.cwd() -def test_base_home(): +def test_base_home() -> None: with pytest.raises(NotImplementedError): Pathy.home() -def test_base_expanduser(): +def test_base_expanduser() -> None: path = Pathy("/fake-bucket/fake-key") with pytest.raises(NotImplementedError): path.expanduser() -def test_base_chmod(): +def test_base_chmod() -> None: path = Pathy("/fake-bucket/fake-key") with pytest.raises(NotImplementedError): path.chmod(0o666) -def test_base_lchmod(): +def test_base_lchmod() -> None: path = Pathy("/fake-bucket/fake-key") with pytest.raises(NotImplementedError): path.lchmod(0o666) -def test_base_group(): +def test_base_group() -> None: path = Pathy("/fake-bucket/fake-key") with pytest.raises(NotImplementedError): path.group() -def test_base_is_mount(): +def test_base_is_mount() -> None: assert not Pathy("/fake-bucket/fake-key").is_mount() -def test_base_is_symlink(): +def test_base_is_symlink() -> None: assert not Pathy("/fake-bucket/fake-key").is_symlink() -def test_base_is_socket(): +def test_base_is_socket() -> None: assert not Pathy("/fake-bucket/fake-key").is_socket() -def test_base_is_fifo(): +def test_base_is_fifo() -> None: assert not Pathy("/fake-bucket/fake-key").is_fifo() -def test_base_is_block_device(): +def test_base_is_block_device() -> None: path = Pathy("/fake-bucket/fake-key") with pytest.raises(NotImplementedError): path.is_block_device() -def test_base_is_char_device(): +def test_base_is_char_device() -> None: path = Pathy("/fake-bucket/fake-key") with pytest.raises(NotImplementedError): path.is_char_device() -def test_base_lstat(): +def test_base_lstat() -> None: path = Pathy("/fake-bucket/fake-key") with pytest.raises(NotImplementedError): path.lstat() -def test_base_symlink_to(): +def test_base_symlink_to() -> None: path = Pathy("/fake-bucket/fake-key") with pytest.raises(NotImplementedError): path.symlink_to("file_name") -def test_base_paths_of_a_different_flavour(): +def test_base_paths_of_a_different_flavour() -> None: with pytest.raises(TypeError): PurePathy("/bucket/key") < PurePosixPath("/bucket/key") @@ -121,7 +122,7 @@ def test_base_paths_of_a_different_flavour(): PureWindowsPath("/bucket/key") > PurePathy("/bucket/key") -def test_base_repr(): +def test_base_repr() -> None: a = PurePathy("/var/tests/fake") assert a.as_posix() == "/var/tests/fake" assert repr(PurePathy("fake_file.txt")) == "PurePathy('fake_file.txt')" @@ -129,7 +130,7 @@ def test_base_repr(): assert bytes(PurePathy("fake_file.txt")) == b"fake_file.txt" -def test_base_scheme_extraction(): +def test_base_scheme_extraction() -> None: assert PurePathy("gs://var/tests/fake").scheme == "gs" assert PurePathy("s3://var/tests/fake").scheme == "s3" assert PurePathy("file://var/tests/fake").scheme == "file" @@ -137,27 +138,27 @@ def test_base_scheme_extraction(): @pytest.mark.skipif(sys.version_info < (3, 6), reason="requires python3.6 or higher") -def test_base_fspath(): +def test_base_fspath() -> None: assert os.fspath(PurePathy("/var/tests/fake")) == "/var/tests/fake" -def test_base_join_strs(): +def test_base_join_strs() -> None: assert PurePathy("foo", "some/path", "bar") == PurePathy("foo/some/path/bar") -def test_base_join_paths(): +def test_base_join_paths() -> None: assert PurePathy(Path("foo"), Path("bar")) == PurePathy("foo/bar") -def test_base_empty(): +def test_base_empty() -> None: assert PurePathy() == PurePathy(".") -def test_base_absolute_paths(): +def test_base_absolute_paths() -> None: assert PurePathy("/etc", "/usr", "lib64") == PurePathy("/usr/lib64") -def test_base_slashes_single_double_dots(): +def test_base_slashes_single_double_dots() -> None: assert PurePathy("foo//bar") == PurePathy("foo/bar") assert PurePathy("foo/./bar") == PurePathy("foo/bar") assert PurePathy("foo/../bar") == PurePathy("bar") @@ -165,12 +166,12 @@ def test_base_slashes_single_double_dots(): assert PurePathy("foo", "../bar") == PurePathy("bar") -def test_base_operators(): +def test_base_operators() -> None: assert PurePathy("/etc") / "init.d" / "apache2" == PurePathy("/etc/init.d/apache2") assert "/var" / PurePathy("tests") / "fake" == PurePathy("/var/tests/fake") -def test_base_parts(): +def test_base_parts() -> None: assert PurePathy("../bar").parts == ("..", "bar") assert PurePathy("foo//bar").parts == ("foo", "bar") assert PurePathy("foo/./bar").parts == ("foo", "bar") @@ -179,7 +180,7 @@ def test_base_parts(): assert PurePathy("/foo/bar").parts == ("/", "foo", "bar") -def test_base_drive(): +def test_base_drive() -> None: assert PurePathy("foo//bar").drive == "" assert PurePathy("foo/./bar").drive == "" assert PurePathy("foo/../bar").drive == "" @@ -188,7 +189,7 @@ def test_base_drive(): assert PurePathy("/foo/bar").drive == "" -def test_base_root(): +def test_base_root() -> None: assert PurePathy("foo//bar").root == "" assert PurePathy("foo/./bar").root == "" assert PurePathy("foo/../bar").root == "" @@ -197,7 +198,7 @@ def test_base_root(): assert PurePathy("/foo/bar").root == "/" -def test_base_anchor(): +def test_base_anchor() -> None: assert PurePathy("foo//bar").anchor == "" assert PurePathy("foo/./bar").anchor == "" assert PurePathy("foo/../bar").anchor == "" @@ -206,7 +207,7 @@ def test_base_anchor(): assert PurePathy("/foo/bar").anchor == "/" -def test_base_parents(): +def test_base_parents() -> None: assert tuple(PurePathy("foo//bar").parents) == ( PurePathy("foo"), PurePathy("."), @@ -224,7 +225,7 @@ def test_base_parents(): ) -def test_base_parent(): +def test_base_parent() -> None: assert PurePathy("foo//bar").parent == PurePathy("foo") assert PurePathy("foo/./bar").parent == PurePathy("foo") assert PurePathy("foo/../bar").parent == PurePathy(".") @@ -235,45 +236,45 @@ def test_base_parent(): assert PurePathy("/").parent == PurePathy("/") -def test_base_name(): +def test_base_name() -> None: assert PurePathy("my/library/fake_file.txt").name == "fake_file.txt" -def test_base_suffix(): +def test_base_suffix() -> None: assert PurePathy("my/library/fake_file.txt").suffix == ".txt" assert PurePathy("my/library.tar.gz").suffix == ".gz" assert PurePathy("my/library").suffix == "" -def test_base_suffixes(): +def test_base_suffixes() -> None: assert PurePathy("my/library.tar.gar").suffixes == [".tar", ".gar"] assert PurePathy("my/library.tar.gz").suffixes == [".tar", ".gz"] assert PurePathy("my/library").suffixes == [] -def test_base_stem(): +def test_base_stem() -> None: assert PurePathy("my/library.tar.gar").stem == "library.tar" assert PurePathy("my/library.tar").stem == "library" assert PurePathy("my/library").stem == "library" -def test_base_uri(): +def test_base_uri() -> None: assert PurePathy("/etc/passwd").as_uri() == "gs://etc/passwd" assert PurePathy("/etc/init.d/apache2").as_uri() == "gs://etc/init.d/apache2" assert PurePathy("/bucket/key").as_uri() == "gs://bucket/key" -def test_base_absolute(): +def test_base_absolute() -> None: assert PurePathy("/a/b").is_absolute() assert not PurePathy("a/b").is_absolute() -def test_base_reserved(): +def test_base_reserved() -> None: assert not PurePathy("/a/b").is_reserved() assert not PurePathy("a/b").is_reserved() -def test_base_joinpath(): +def test_base_joinpath() -> None: assert PurePathy("/etc").joinpath("passwd") == PurePathy("/etc/passwd") assert PurePathy("/etc").joinpath(PurePathy("passwd")) == PurePathy("/etc/passwd") assert PurePathy("/etc").joinpath("init.d", "apache2") == PurePathy( @@ -281,7 +282,7 @@ def test_base_joinpath(): ) -def test_base_match(): +def test_base_match() -> None: assert PurePathy("a/b.py").match("*.py") assert PurePathy("/a/b/c.py").match("b/*.py") assert not PurePathy("/a/b/c.py").match("a/*.py") @@ -290,7 +291,7 @@ def test_base_match(): assert not PurePathy("a/b.py").match("*.Py") -def test_base_relative_to(): +def test_base_relative_to() -> None: gcs_path = PurePathy("/etc/passwd") assert gcs_path.relative_to("/") == PurePathy("etc/passwd") assert gcs_path.relative_to("/etc") == PurePathy("passwd") @@ -298,7 +299,7 @@ def test_base_relative_to(): gcs_path.relative_to("/usr") -def test_base_with_name(): +def test_base_with_name() -> None: gcs_path = PurePathy("/Downloads/pathlib.tar.gz") assert gcs_path.with_name("fake_file.txt") == PurePathy("/Downloads/fake_file.txt") gcs_path = PurePathy("/") @@ -306,7 +307,7 @@ def test_base_with_name(): gcs_path.with_name("fake_file.txt") -def test_base_with_suffix(): +def test_base_with_suffix() -> None: gcs_path = PurePathy("/Downloads/pathlib.tar.gz") assert gcs_path.with_suffix(".bz2") == PurePathy("/Downloads/pathlib.tar.bz2") gcs_path = PurePathy("README") @@ -315,29 +316,29 @@ def test_base_with_suffix(): assert gcs_path.with_suffix("") == PurePathy("README") -def test_api_path_support(): +def test_api_path_support() -> None: assert PurePathy in Pathy.mro() # type: ignore assert Path in Pathy.mro() # type: ignore @pytest.mark.parametrize("adapter", TEST_ADAPTERS) -def test_api_is_path_instance(with_adapter): +def test_api_is_path_instance(with_adapter: str) -> None: blob = Pathy("gs://fake/blob") assert isinstance(blob, Path) @pytest.mark.parametrize("adapter", TEST_ADAPTERS) -def test_api_fluid(with_adapter, bucket: str): +def test_api_fluid(with_adapter: str, bucket: str) -> None: path: FluidPath = Pathy.fluid(f"gs://{bucket}/fake-key") assert isinstance(path, Pathy) - path: FluidPath = Pathy.fluid("foo/bar.txt") + path = Pathy.fluid("foo/bar.txt") assert isinstance(path, Path) - path: FluidPath = Pathy.fluid("/dev/null") + path = Pathy.fluid("/dev/null") assert isinstance(path, Path) @pytest.mark.parametrize("adapter", TEST_ADAPTERS) -def test_api_path_to_local(with_adapter, bucket: str): +def test_api_path_to_local(with_adapter: str, bucket: str) -> None: root: Pathy = Pathy.from_bucket(bucket) / "to_local" foo_blob: Pathy = root / "foo" foo_blob.write_text("---") @@ -377,7 +378,7 @@ def test_api_path_to_local(with_adapter, bucket: str): @pytest.mark.parametrize("adapter", TEST_ADAPTERS) -def test_api_stat(with_adapter, bucket: str): +def test_api_stat(with_adapter: str, bucket: str) -> None: path = Pathy("fake-bucket-1234-0987/fake-key") with pytest.raises(ValueError): path.stat() @@ -385,8 +386,8 @@ def test_api_stat(with_adapter, bucket: str): path.write_text("a-a-a-a-a-a-a") stat = path.stat() assert isinstance(stat, BlobStat) - assert stat.size > 0 - assert stat.last_modified > 0 + assert stat.size is not None and stat.size > 0 + assert stat.last_modified is not None and stat.last_modified > 0 with pytest.raises(ValueError): assert Pathy(f"gs://{bucket}").stat() with pytest.raises(FileNotFoundError): @@ -394,7 +395,7 @@ def test_api_stat(with_adapter, bucket: str): @pytest.mark.parametrize("adapter", TEST_ADAPTERS) -def test_api_resolve(with_adapter, bucket: str): +def test_api_resolve(with_adapter: str, bucket: str) -> None: path = Pathy(f"gs://{bucket}/fake-key") assert path.resolve() == path path = Pathy(f"gs://{bucket}/dir/../fake-key") @@ -402,7 +403,7 @@ def test_api_resolve(with_adapter, bucket: str): @pytest.mark.parametrize("adapter", TEST_ADAPTERS) -def test_api_exists(with_adapter, bucket: str): +def test_api_exists(with_adapter: str, bucket: str) -> None: path = Pathy("./fake-key") with pytest.raises(ValueError): path.exists() @@ -423,7 +424,7 @@ def test_api_exists(with_adapter, bucket: str): @pytest.mark.parametrize("adapter", TEST_ADAPTERS) -def test_api_glob(with_adapter, bucket: str): +def test_api_glob(with_adapter: str, bucket: str) -> None: for i in range(3): path = Pathy(f"gs://{bucket}/glob/{i}.file") path.write_text("---") @@ -456,7 +457,7 @@ def test_api_glob(with_adapter, bucket: str): @pytest.mark.parametrize("adapter", TEST_ADAPTERS) -def test_api_unlink_path(with_adapter, bucket: str): +def test_api_unlink_path(with_adapter: str, bucket: str) -> None: path = Pathy(f"gs://{bucket}/unlink/404.txt") with pytest.raises(FileNotFoundError): path.unlink() @@ -468,7 +469,7 @@ def test_api_unlink_path(with_adapter, bucket: str): @pytest.mark.parametrize("adapter", TEST_ADAPTERS) -def test_api_is_dir(with_adapter, bucket: str): +def test_api_is_dir(with_adapter: str, bucket: str) -> None: path = Pathy(f"gs://{bucket}/is_dir/subfolder/another/my.file") path.write_text("---") assert path.is_dir() is False @@ -477,7 +478,7 @@ def test_api_is_dir(with_adapter, bucket: str): @pytest.mark.parametrize("adapter", TEST_ADAPTERS) -def test_api_is_file(with_adapter, bucket: str): +def test_api_is_file(with_adapter: str, bucket: str) -> None: path = Pathy(f"gs://{bucket}/is_file/subfolder/another/my.file") path.write_text("---") # The full file is a file @@ -488,7 +489,7 @@ def test_api_is_file(with_adapter, bucket: str): @pytest.mark.parametrize("adapter", TEST_ADAPTERS) -def test_api_iterdir(with_adapter, bucket: str): +def test_api_iterdir(with_adapter: str, bucket: str) -> None: # (n) files in a folder for i in range(2): path = Pathy(f"gs://{bucket}/iterdir/{i}.file") @@ -508,7 +509,7 @@ def test_api_iterdir(with_adapter, bucket: str): @pytest.mark.parametrize("adapter", TEST_ADAPTERS) -def test_api_iterdir_pipstore(with_adapter, bucket: str): +def test_api_iterdir_pipstore(with_adapter: str, bucket: str) -> None: path = Pathy.from_bucket(bucket) / "iterdir_pipstore/prodigy/prodigy.whl" path.write_bytes(b"---") path = Pathy.from_bucket(bucket) / "iterdir_pipstore" @@ -517,7 +518,7 @@ def test_api_iterdir_pipstore(with_adapter, bucket: str): @pytest.mark.parametrize("adapter", TEST_ADAPTERS) -def test_api_open_for_read(with_adapter, bucket: str): +def test_api_open_for_read(with_adapter: str, bucket: str) -> None: path = Pathy(f"gs://{bucket}/read/file.txt") path.write_text("---") with path.open() as file_obj: @@ -526,7 +527,7 @@ def test_api_open_for_read(with_adapter, bucket: str): @pytest.mark.parametrize("adapter", TEST_ADAPTERS) -def test_api_open_for_write(with_adapter, bucket: str): +def test_api_open_for_write(with_adapter: str, bucket: str) -> None: path = Pathy(f"gs://{bucket}/write/file.txt") with path.open(mode="w") as file_obj: file_obj.write("---") @@ -537,7 +538,7 @@ def test_api_open_for_write(with_adapter, bucket: str): @pytest.mark.parametrize("adapter", TEST_ADAPTERS) -def test_api_open_binary_read(with_adapter, bucket: str): +def test_api_open_binary_read(with_adapter: str, bucket: str) -> None: path = Pathy(f"gs://{bucket}/read_binary/file.txt") path.write_bytes(b"---") with path.open(mode="rb") as file_obj: @@ -548,7 +549,7 @@ def test_api_open_binary_read(with_adapter, bucket: str): @pytest.mark.parametrize("adapter", TEST_ADAPTERS) -def test_api_readwrite_text(with_adapter, bucket: str): +def test_api_readwrite_text(with_adapter: str, bucket: str) -> None: path = Pathy(f"gs://{bucket}/write_text/file.txt") path.write_text("---") with path.open() as file_obj: @@ -557,14 +558,14 @@ def test_api_readwrite_text(with_adapter, bucket: str): @pytest.mark.parametrize("adapter", TEST_ADAPTERS) -def test_api_readwrite_bytes(with_adapter, bucket: str): +def test_api_readwrite_bytes(with_adapter: str, bucket: str) -> None: path = Pathy(f"gs://{bucket}/write_bytes/file.txt") path.write_bytes(b"---") assert path.read_bytes() == b"---" @pytest.mark.parametrize("adapter", TEST_ADAPTERS) -def test_api_readwrite_lines(with_adapter, bucket: str): +def test_api_readwrite_lines(with_adapter: str, bucket: str) -> None: path = Pathy(f"gs://{bucket}/write_text/file.txt") with path.open("w") as file_obj: file_obj.writelines(["---"]) @@ -575,7 +576,7 @@ def test_api_readwrite_lines(with_adapter, bucket: str): @pytest.mark.parametrize("adapter", TEST_ADAPTERS) -def test_api_owner(with_adapter, bucket: str): +def test_api_owner(with_adapter: str, bucket: str) -> None: # Raises for invalid file with pytest.raises(FileNotFoundError): Pathy(f"gs://{bucket}/write_text/not_a_valid_blob").owner() @@ -593,7 +594,7 @@ def test_api_owner(with_adapter, bucket: str): @pytest.mark.parametrize("adapter", TEST_ADAPTERS) -def test_api_rename_files_in_bucket(with_adapter, bucket: str): +def test_api_rename_files_in_bucket(with_adapter: str, bucket: str) -> None: # Rename a single file Pathy(f"gs://{bucket}/rename/file.txt").write_text("---") Pathy(f"gs://{bucket}/rename/file.txt").rename(f"gs://{bucket}/rename/other.txt") @@ -602,7 +603,9 @@ def test_api_rename_files_in_bucket(with_adapter, bucket: str): @pytest.mark.parametrize("adapter", TEST_ADAPTERS) -def test_api_rename_files_across_buckets(with_adapter, bucket: str, other_bucket: str): +def test_api_rename_files_across_buckets( + with_adapter: str, bucket: str, other_bucket: str +) -> None: # Rename a single file across buckets Pathy(f"gs://{bucket}/rename/file.txt").write_text("---") Pathy(f"gs://{bucket}/rename/file.txt").rename( @@ -613,7 +616,7 @@ def test_api_rename_files_across_buckets(with_adapter, bucket: str, other_bucket @pytest.mark.parametrize("adapter", TEST_ADAPTERS) -def test_api_rename_folders_in_bucket(with_adapter, bucket: str): +def test_api_rename_folders_in_bucket(with_adapter: str, bucket: str) -> None: # Rename a folder in the same bucket Pathy(f"gs://{bucket}/rename/folder/one.txt").write_text("---") Pathy(f"gs://{bucket}/rename/folder/two.txt").write_text("---") @@ -628,8 +631,8 @@ def test_api_rename_folders_in_bucket(with_adapter, bucket: str): @pytest.mark.parametrize("adapter", TEST_ADAPTERS) def test_api_rename_folders_across_buckets( - with_adapter, bucket: str, other_bucket: str -): + with_adapter: str, bucket: str, other_bucket: str +) -> None: # Rename a folder across buckets Pathy(f"gs://{bucket}/rename/folder/one.txt").write_text("---") Pathy(f"gs://{bucket}/rename/folder/two.txt").write_text("---") @@ -643,7 +646,7 @@ def test_api_rename_folders_across_buckets( @pytest.mark.parametrize("adapter", TEST_ADAPTERS) -def test_api_replace_files_in_bucket(with_adapter, bucket: str): +def test_api_replace_files_in_bucket(with_adapter: str, bucket: str) -> None: # replace a single file Pathy(f"gs://{bucket}/replace/file.txt").write_text("---") Pathy(f"gs://{bucket}/replace/file.txt").replace(f"gs://{bucket}/replace/other.txt") @@ -652,7 +655,9 @@ def test_api_replace_files_in_bucket(with_adapter, bucket: str): @pytest.mark.parametrize("adapter", TEST_ADAPTERS) -def test_api_replace_files_across_buckets(with_adapter, bucket: str, other_bucket: str): +def test_api_replace_files_across_buckets( + with_adapter: str, bucket: str, other_bucket: str +) -> None: # Rename a single file across buckets Pathy(f"gs://{bucket}/replace/file.txt").write_text("---") Pathy(f"gs://{bucket}/replace/file.txt").replace( @@ -663,7 +668,7 @@ def test_api_replace_files_across_buckets(with_adapter, bucket: str, other_bucke @pytest.mark.parametrize("adapter", TEST_ADAPTERS) -def test_api_replace_folders_in_bucket(with_adapter, bucket: str): +def test_api_replace_folders_in_bucket(with_adapter: str, bucket: str) -> None: # Rename a folder in the same bucket Pathy(f"gs://{bucket}/replace/folder/one.txt").write_text("---") Pathy(f"gs://{bucket}/replace/folder/two.txt").write_text("---") @@ -678,8 +683,8 @@ def test_api_replace_folders_in_bucket(with_adapter, bucket: str): @pytest.mark.parametrize("adapter", TEST_ADAPTERS) def test_api_replace_folders_across_buckets( - with_adapter, bucket: str, other_bucket: str -): + with_adapter: str, bucket: str, other_bucket: str +) -> None: # Rename a folder across buckets Pathy(f"gs://{bucket}/replace/folder/one.txt").write_text("---") Pathy(f"gs://{bucket}/replace/folder/two.txt").write_text("---") @@ -693,7 +698,7 @@ def test_api_replace_folders_across_buckets( @pytest.mark.parametrize("adapter", TEST_ADAPTERS) -def test_api_rmdir(with_adapter, bucket: str): +def test_api_rmdir(with_adapter: str, bucket: str) -> None: Pathy(f"gs://{bucket}/rmdir/one.txt").write_text("---") Pathy(f"gs://{bucket}/rmdir/folder/two.txt").write_text("---") path = Pathy(f"gs://{bucket}/rmdir/") @@ -704,7 +709,7 @@ def test_api_rmdir(with_adapter, bucket: str): @pytest.mark.parametrize("adapter", TEST_ADAPTERS) -def test_api_rglob_unlink(with_adapter, bucket: str): +def test_api_rglob_unlink(with_adapter: str, bucket: str) -> None: files = [f"gs://{bucket}/rglob_and_unlink/{i}.file.txt" for i in range(3)] for file in files: Pathy(file).write_text("---") @@ -719,7 +724,7 @@ def test_api_rglob_unlink(with_adapter, bucket: str): @pytest.mark.parametrize("adapter", TEST_ADAPTERS) -def test_api_mkdir(with_adapter, bucket: str): +def test_api_mkdir(with_adapter: str, bucket: str) -> None: bucket_name = f"pathy-e2e-test-{uuid4().hex}" # Create a bucket path = Pathy(f"gs://{bucket_name}/") @@ -742,7 +747,7 @@ def test_api_mkdir(with_adapter, bucket: str): @pytest.mark.parametrize("adapter", TEST_ADAPTERS) -def test_api_ignore_extension(with_adapter, bucket: str): +def test_api_ignore_extension(with_adapter: str, bucket: str) -> None: """The smart_open library does automatic decompression based on the filename. We disable that to avoid errors, e.g. if you have a .tar.gz file that isn't gzipped.""" @@ -753,7 +758,9 @@ def test_api_ignore_extension(with_adapter, bucket: str): assert again is not None -def test_api_raises_with_no_known_bucket_clients_for_a_scheme(temp_folder): +def test_api_raises_with_no_known_bucket_clients_for_a_scheme( + temp_folder: Path, +) -> None: accessor = BucketsAccessor() path = Pathy("foo://foo") with pytest.raises(ValueError): @@ -764,7 +771,7 @@ def test_api_raises_with_no_known_bucket_clients_for_a_scheme(temp_folder): @pytest.mark.skip("requires: https://github.com/explosion/thinc/pull/465") -def test_api_export_spacy_model(temp_folder): +def test_api_export_spacy_model(temp_folder: Path) -> None: """spaCy model loading is one of the things we need to support""" use_fs(temp_folder) bucket = Pathy("gs://my-bucket/") @@ -781,7 +788,7 @@ def test_api_export_spacy_model(temp_folder): assert sorted_entries == expected_entries -def test_client_create_bucket(temp_folder: Path): +def test_client_create_bucket(temp_folder: Path) -> None: bucket_target = temp_folder / "foo" assert bucket_target.exists() is False cl = BucketClientFS(temp_folder) @@ -789,9 +796,9 @@ def test_client_create_bucket(temp_folder: Path): assert bucket_target.exists() is True -def test_client_base_bucket_raises_not_implemented(): +def test_client_base_bucket_raises_not_implemented() -> None: bucket = Bucket() - blob = Blob(bucket, "foo", -1, -1, None, None) + blob: Blob = Blob(bucket, "foo", -1, -1, None, None) with pytest.raises(NotImplementedError): bucket.copy_blob(blob, bucket, "baz") with pytest.raises(NotImplementedError): @@ -804,15 +811,15 @@ def test_client_base_bucket_raises_not_implemented(): bucket.exists() -def test_client_base_blob_raises_not_implemented(): - blob = Blob(Bucket(), "foo", -1, -1, None, None) +def test_client_base_blob_raises_not_implemented() -> None: + blob: Blob = Blob(Bucket(), "foo", -1, -1, None, None) with pytest.raises(NotImplementedError): blob.delete() with pytest.raises(NotImplementedError): blob.exists() -def test_client_base_bucket_client_raises_not_implemented(): +def test_client_base_bucket_client_raises_not_implemented() -> None: client = BucketClient() with pytest.raises(NotImplementedError): client.lookup_bucket(Pathy("gs://foo")) diff --git a/pathy/tests/test_cli.py b/pathy/tests/test_cli.py index 91c2643..446d5c8 100644 --- a/pathy/tests/test_cli.py +++ b/pathy/tests/test_cli.py @@ -13,7 +13,7 @@ @pytest.mark.parametrize("adapter", TEST_ADAPTERS) -def test_cli_cp_file(with_adapter, bucket: str): +def test_cli_cp_file(with_adapter: str, bucket: str) -> None: source = f"gs://{bucket}/cli_cp_file/file.txt" destination = f"gs://{bucket}/cli_cp_file/other.txt" Pathy(source).write_text("---") @@ -23,7 +23,7 @@ def test_cli_cp_file(with_adapter, bucket: str): @pytest.mark.parametrize("adapter", TEST_ADAPTERS) -def test_cli_cp_folder(with_adapter, bucket: str): +def test_cli_cp_folder(with_adapter: str, bucket: str) -> None: root = Pathy.from_bucket(bucket) source = root / "cli_cp_folder" destination = root / "cli_cp_folder_other" @@ -39,7 +39,7 @@ def test_cli_cp_folder(with_adapter, bucket: str): @pytest.mark.parametrize("adapter", TEST_ADAPTERS) -def test_cli_mv_folder(with_adapter, bucket: str): +def test_cli_mv_folder(with_adapter: str, bucket: str) -> None: root = Pathy.from_bucket(bucket) source = root / "cli_mv_folder" destination = root / "cli_mv_folder_other" @@ -60,7 +60,7 @@ def test_cli_mv_folder(with_adapter, bucket: str): @pytest.mark.parametrize("adapter", TEST_ADAPTERS) -def test_cli_mv_file(with_adapter, bucket: str): +def test_cli_mv_file(with_adapter: str, bucket: str) -> None: source = f"gs://{bucket}/cli_mv_file/file.txt" destination = f"gs://{bucket}/cli_mv_file/other.txt" Pathy(source).write_text("---") @@ -71,7 +71,9 @@ def test_cli_mv_file(with_adapter, bucket: str): @pytest.mark.parametrize("adapter", TEST_ADAPTERS) -def test_cli_mv_file_across_buckets(with_adapter, bucket: str, other_bucket: str): +def test_cli_mv_file_across_buckets( + with_adapter: str, bucket: str, other_bucket: str +) -> None: source = f"gs://{bucket}/cli_mv_file_across_buckets/file.txt" destination = f"gs://{other_bucket}/cli_mv_file_across_buckets/other.txt" Pathy(source).write_text("---") @@ -82,7 +84,9 @@ def test_cli_mv_file_across_buckets(with_adapter, bucket: str, other_bucket: str @pytest.mark.parametrize("adapter", TEST_ADAPTERS) -def test_cli_mv_folder_across_buckets(with_adapter, bucket: str, other_bucket: str): +def test_cli_mv_folder_across_buckets( + with_adapter: str, bucket: str, other_bucket: str +) -> None: source = Pathy.from_bucket(bucket) / "cli_mv_folder_across_buckets" destination = Pathy.from_bucket(other_bucket) / "cli_mv_folder_across_buckets" for i in range(2): @@ -102,7 +106,7 @@ def test_cli_mv_folder_across_buckets(with_adapter, bucket: str, other_bucket: s @pytest.mark.parametrize("adapter", TEST_ADAPTERS) -def test_cli_rm_file(with_adapter, bucket: str): +def test_cli_rm_file(with_adapter: str, bucket: str) -> None: source = f"gs://{bucket}/cli_rm_file/file.txt" path = Pathy(source) path.write_text("---") @@ -112,7 +116,7 @@ def test_cli_rm_file(with_adapter, bucket: str): @pytest.mark.parametrize("adapter", TEST_ADAPTERS) -def test_cli_rm_verbose(with_adapter, bucket: str): +def test_cli_rm_verbose(with_adapter: str, bucket: str) -> None: root = Pathy.from_bucket(bucket) / "cli_rm_folder" source = str(root / "file.txt") other = str(root / "folder/other") @@ -131,7 +135,7 @@ def test_cli_rm_verbose(with_adapter, bucket: str): @pytest.mark.parametrize("adapter", TEST_ADAPTERS) -def test_cli_rm_folder(with_adapter, bucket: str): +def test_cli_rm_folder(with_adapter: str, bucket: str) -> None: root = Pathy.from_bucket(bucket) source = root / "cli_rm_folder" for i in range(2): @@ -149,7 +153,7 @@ def test_cli_rm_folder(with_adapter, bucket: str): @pytest.mark.parametrize("adapter", TEST_ADAPTERS) -def test_cli_ls(with_adapter, bucket: str): +def test_cli_ls(with_adapter: str, bucket: str) -> None: root = Pathy.from_bucket(bucket) / "cli_ls" one = str(root / "file.txt") two = str(root / "other.txt") diff --git a/pathy/tests/test_clients.py b/pathy/tests/test_clients.py index 88cdfe2..3a112bc 100644 --- a/pathy/tests/test_clients.py +++ b/pathy/tests/test_clients.py @@ -1,5 +1,6 @@ import time from pathlib import Path +from typing import Any import pytest @@ -18,24 +19,24 @@ from .conftest import TEST_ADAPTERS -def test_clients_get_client_works_with_builtin_schems(): +def test_clients_get_client_works_with_builtin_schems() -> None: assert isinstance(get_client("gs"), BucketClientGCS) assert isinstance(get_client("file"), BucketClientFS) assert isinstance(get_client(""), BucketClientFS) -def test_clients_get_client_respects_use_fs_override(): +def test_clients_get_client_respects_use_fs_override() -> None: use_fs(True) assert isinstance(get_client("gs"), BucketClientFS) use_fs(False) -def test_clients_get_client_errors_with_unknown_scheme(): +def test_clients_get_client_errors_with_unknown_scheme() -> None: with pytest.raises(ValueError): get_client("foo") -def test_clients_use_fs(with_fs: Path): +def test_clients_use_fs(with_fs: Path) -> None: assert get_fs_client() is None # Use the default path use_fs() @@ -64,7 +65,7 @@ def test_clients_use_fs(with_fs: Path): @pytest.mark.parametrize("adapter", TEST_ADAPTERS) -def test_api_use_fs_cache(with_adapter, with_fs: str, bucket: str): +def test_api_use_fs_cache(with_adapter: str, with_fs: str, bucket: str) -> None: path = Pathy(f"gs://{bucket}/directory/foo.txt") path.write_text("---") assert isinstance(path, Pathy) @@ -97,21 +98,21 @@ class BucketClientTest(BucketClient): def __init__(self, required_arg: bool) -> None: self.recreate(required_arg=required_arg) - def recreate(self, required_arg: bool) -> None: - self.required_arg = required_arg + def recreate(self, **kwargs: Any) -> None: + self.required_arg = kwargs["required_arg"] -def test_clients_set_client_params(): +def test_clients_set_client_params() -> None: register_client("test", BucketClientTest) with pytest.raises(TypeError): get_client("test") set_client_params("test", required_arg=True) - client = get_client("test") + client: BucketClientTest = get_client("test") assert isinstance(client, BucketClientTest) -def test_clients_set_client_params_recreates_client(): +def test_clients_set_client_params_recreates_client() -> None: register_client("test", BucketClientTest) set_client_params("test", required_arg=False) client: BucketClientTest = get_client("test") diff --git a/pathy/tests/test_file.py b/pathy/tests/test_file.py index 5d88eaa..e91c580 100644 --- a/pathy/tests/test_file.py +++ b/pathy/tests/test_file.py @@ -1,5 +1,5 @@ import pathlib -from typing import Optional +from typing import Any, Optional import mock import pytest @@ -8,13 +8,13 @@ from pathy.file import BlobFS, BucketClientFS, BucketFS -def raise_owner(self): +def raise_owner(self: Any) -> None: raise KeyError("duh") @pytest.mark.parametrize("adapter", ["fs"]) @mock.patch.object(pathlib.Path, "owner", raise_owner) -def test_file_get_blob_owner_key_error_protection(with_adapter): +def test_file_get_blob_owner_key_error_protection(with_adapter: str) -> None: gs_bucket = Pathy("gs://my_bucket") gs_bucket.mkdir() path = gs_bucket / "blob.txt" @@ -22,4 +22,4 @@ def test_file_get_blob_owner_key_error_protection(with_adapter): gcs_client: BucketClientFS = get_client("gs") bucket: BucketFS = gcs_client.get_bucket(gs_bucket) blob: Optional[BlobFS] = bucket.get_blob("blob.txt") - assert blob.owner is None + assert blob is not None and blob.owner is None diff --git a/tools/format.sh b/tools/format.sh index b9e08cb..79cacd5 100644 --- a/tools/format.sh +++ b/tools/format.sh @@ -2,7 +2,7 @@ . .env/bin/activate # Sort imports one per line, so autoflake can remove unused imports -isort pathy tests --force-single-line-imports -autoflake --remove-all-unused-imports --recursive --remove-unused-variables --in-place pathy tests --exclude=__init__.py -isort pathy tests -black pathy tests \ No newline at end of file +isort pathy --force-single-line-imports +autoflake --remove-all-unused-imports --recursive --remove-unused-variables --in-place pathy --exclude=__init__.py +isort pathy +black pathy \ No newline at end of file diff --git a/tools/lint.sh b/tools/lint.sh index 84665a2..4556ab6 100644 --- a/tools/lint.sh +++ b/tools/lint.sh @@ -6,8 +6,8 @@ set -e echo "========================= mypy" mypy pathy echo "========================= flake8" -flake8 pathy tests +flake8 pathy echo "========================= black" -black pathy tests --check +black pathy --check echo "========================= pyright" -npx pyright pathy tests \ No newline at end of file +npx pyright pathy \ No newline at end of file From 22f199eb86ead3b03f790427a0a1d7f7bdd4c3d7 Mon Sep 17 00:00:00 2001 From: justindujardin Date: Tue, 2 Feb 2021 13:22:31 -0800 Subject: [PATCH 3/4] chore: only build master branch --- .travis.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index bc168a9..4dd9092 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ matrix: python: 3.7 before_install: - nvm install 12 - - nvm use 12 + - nvm use 12 before_script: - npm install && npx ts-node tools/ci-set-build-version.ts - sh tools/setup.sh @@ -21,5 +21,5 @@ cache: notifications: email: false branches: - except: - - /^v\d+\.\d+\.\d+$/ + only: + - master From 3a3f199bba74bdc49e7f34d1eb3ff7bd1d329ac8 Mon Sep 17 00:00:00 2001 From: justindujardin Date: Tue, 9 Feb 2021 13:47:40 -0800 Subject: [PATCH 4/4] chore: also build develop branch --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 4dd9092..9d71c03 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,3 +23,4 @@ notifications: branches: only: - master + - develop