From 3f805fe9f8e37794b5f5d9cce125057dd6e186f9 Mon Sep 17 00:00:00 2001 From: Brian Hannafious <32105697+xbrianh@users.noreply.github.com> Date: Fri, 11 Oct 2024 21:23:46 -0700 Subject: [PATCH] Fix absloc url --- tests/test_xdlake.py | 2 +- tests/test_xdlake_cloud.py | 7 +++++++ xdlake/storage.py | 8 +++++++- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/tests/test_xdlake.py b/tests/test_xdlake.py index ff93b33..78204c7 100644 --- a/tests/test_xdlake.py +++ b/tests/test_xdlake.py @@ -339,7 +339,7 @@ def test_file_uris(self): xdl = xdl.write(self.gen_table()) self.assertEqual(number_of_writes, len(xdl.file_uris())) for uri in xdl.file_uris(): - self.assertTrue(uri.startswith("file://")) + xdl.loc.fs.head(uri, size=1) if __name__ == '__main__': unittest.main() diff --git a/tests/test_xdlake_cloud.py b/tests/test_xdlake_cloud.py index 98b3cba..20147d1 100644 --- a/tests/test_xdlake_cloud.py +++ b/tests/test_xdlake_cloud.py @@ -28,6 +28,10 @@ def test_remote_log(self): self._test_clone(xdl) self._test_delete(xdl) + def _test_file_uris(self, xdl: xdlake.DeltaTable): + for url in xdl.file_uris(): + xdl.loc.fs.head(url, size=1) + def test_s3(self): partition_by = self.partition_by[:1] arrow_tables = [self.gen_table() for _ in range(3)] @@ -37,6 +41,7 @@ def test_s3(self): assert_arrow_table_equal(pa.concat_tables(arrow_tables), xdl.to_pyarrow_table()) self._test_clone(xdl) self._test_delete(xdl) + self._test_file_uris(xdl) def test_gs(self): partition_by = self.partition_by[:1] @@ -47,6 +52,7 @@ def test_gs(self): assert_arrow_table_equal(pa.concat_tables(arrow_tables), xdl.to_pyarrow_table()) self._test_clone(xdl) self._test_delete(xdl) + self._test_file_uris(xdl) def test_azure_storage(self): partition_by = self.partition_by[:1] @@ -63,6 +69,7 @@ def test_azure_storage(self): xdlake.storage.register_default_filesystem_for_protocol("az", storage_options=storage_options) self._test_clone(xdl) self._test_delete(xdl) + self._test_file_uris(xdl) def test_import_refs(self): paths = [os.path.join(f"{self.scratch_folder}", f"{uuid4()}", f"{uuid4()}.parquet") for _ in range(2)] diff --git a/xdlake/storage.py b/xdlake/storage.py index c5c7f1d..bf2d01e 100644 --- a/xdlake/storage.py +++ b/xdlake/storage.py @@ -90,9 +90,15 @@ class Location: def __init__(self, scheme: str, path: str, storage_options: dict | None = None): self.scheme = scheme self.path = path - self.url = f"{self.scheme}://{self.path}" self.storage_options = storage_options + @property + def url(self): + if self.path.startswith(self.scheme): + return self.path + else: + return f"{self.scheme}://{self.path}" + @classmethod def with_location(cls, loc: str | Any, storage_options: dict | None = None) -> "Location": """Create a Location from a string or Location.