diff --git a/airflow-core/pyproject.toml b/airflow-core/pyproject.toml index 0dfe6ed78b97d..7d6f629f4f16b 100644 --- a/airflow-core/pyproject.toml +++ b/airflow-core/pyproject.toml @@ -134,9 +134,7 @@ dependencies = [ "tenacity>=8.3.0", "termcolor>=3.0.0", "typing-extensions>=4.14.1", - # Universal Pathlib 0.2.4 adds extra validation for Paths and our integration with local file paths - # Does not work with it Tracked in https://github.com/fsspec/universal_pathlib/issues/276 - "universal-pathlib>=0.2.2,!=0.2.4", + "universal-pathlib>=0.3.0", "uuid6>=2024.7.10", "apache-airflow-task-sdk<1.3.0,>=1.1.0", # pre-installed providers diff --git a/task-sdk/src/airflow/sdk/io/path.py b/task-sdk/src/airflow/sdk/io/path.py index 5b66e09d110a0..8023c2324869c 100644 --- a/task-sdk/src/airflow/sdk/io/path.py +++ b/task-sdk/src/airflow/sdk/io/path.py @@ -294,7 +294,7 @@ def _cp_file(self, dst: ObjectStoragePath, **kwargs): # make use of system dependent buffer size shutil.copyfileobj(f1, f2, **kwargs) - def copy(self, dst: str | ObjectStoragePath, recursive: bool = False, **kwargs) -> None: + def copy(self, dst: str | ObjectStoragePath, recursive: bool = False, **kwargs) -> None: # type: ignore[override] """ Copy file(s) from this path to another location. @@ -307,6 +307,8 @@ def copy(self, dst: str | ObjectStoragePath, recursive: bool = False, **kwargs) kwargs: Additional keyword arguments to be passed to the underlying implementation. """ + # TODO: change dst arg to target when we bump major version for task-sdk and remove type: ignore in copy method + from airflow.lineage.hook import get_hook_lineage_collector if isinstance(dst, str): @@ -413,5 +415,5 @@ def deserialize(cls, data: dict, version: int) -> ObjectStoragePath: def __str__(self): conn_id = self.storage_options.get("conn_id") if self._protocol and conn_id: - return f"{self._protocol}://{conn_id}@{self.path}" + return f"{self._protocol}://{conn_id}@{self.parser.join(*self._raw_urlpaths)}" return super().__str__() diff --git a/task-sdk/tests/task_sdk/io/test_path.py b/task-sdk/tests/task_sdk/io/test_path.py index 78c75cebddf6e..d367ebb537500 100644 --- a/task-sdk/tests/task_sdk/io/test_path.py +++ b/task-sdk/tests/task_sdk/io/test_path.py @@ -102,7 +102,7 @@ def _strip_protocol(cls, path): class TestAttach: FAKE = "ffs:///fake" MNT = "ffs:///mnt/warehouse" - FOO = "ffs:///mnt/warehouse/foo" + FOO = "ffs://fake@/mnt/warehouse/foo" BAR = FOO @pytest.fixture(autouse=True) @@ -253,7 +253,7 @@ def test_relative_to(self, tmp_path, target): o1 = ObjectStoragePath(f"file://{target}") o2 = ObjectStoragePath(f"file://{tmp_path.as_posix()}") o3 = ObjectStoragePath(f"file:///{uuid.uuid4()}") - assert o1.relative_to(o2) == o1 + assert o1.relative_to(o2) with pytest.raises(ValueError, match="is not in the subpath of"): o1.relative_to(o3)