Skip to content

Commit

Permalink
Fix absloc url
Browse files Browse the repository at this point in the history
  • Loading branch information
xbrianh committed Oct 12, 2024
1 parent f93502e commit 3f805fe
Show file tree
Hide file tree
Showing 3 changed files with 15 additions and 2 deletions.
2 changes: 1 addition & 1 deletion tests/test_xdlake.py
Original file line number Diff line number Diff line change
Expand Up @@ -339,7 +339,7 @@ def test_file_uris(self):
xdl = xdl.write(self.gen_table())
self.assertEqual(number_of_writes, len(xdl.file_uris()))
for uri in xdl.file_uris():
self.assertTrue(uri.startswith("file://"))
xdl.loc.fs.head(uri, size=1)

if __name__ == '__main__':
unittest.main()
7 changes: 7 additions & 0 deletions tests/test_xdlake_cloud.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,10 @@ def test_remote_log(self):
self._test_clone(xdl)
self._test_delete(xdl)

def _test_file_uris(self, xdl: xdlake.DeltaTable):
for url in xdl.file_uris():
xdl.loc.fs.head(url, size=1)

def test_s3(self):
partition_by = self.partition_by[:1]
arrow_tables = [self.gen_table() for _ in range(3)]
Expand All @@ -37,6 +41,7 @@ def test_s3(self):
assert_arrow_table_equal(pa.concat_tables(arrow_tables), xdl.to_pyarrow_table())
self._test_clone(xdl)
self._test_delete(xdl)
self._test_file_uris(xdl)

def test_gs(self):
partition_by = self.partition_by[:1]
Expand All @@ -47,6 +52,7 @@ def test_gs(self):
assert_arrow_table_equal(pa.concat_tables(arrow_tables), xdl.to_pyarrow_table())
self._test_clone(xdl)
self._test_delete(xdl)
self._test_file_uris(xdl)

def test_azure_storage(self):
partition_by = self.partition_by[:1]
Expand All @@ -63,6 +69,7 @@ def test_azure_storage(self):
xdlake.storage.register_default_filesystem_for_protocol("az", storage_options=storage_options)
self._test_clone(xdl)
self._test_delete(xdl)
self._test_file_uris(xdl)

def test_import_refs(self):
paths = [os.path.join(f"{self.scratch_folder}", f"{uuid4()}", f"{uuid4()}.parquet") for _ in range(2)]
Expand Down
8 changes: 7 additions & 1 deletion xdlake/storage.py
Original file line number Diff line number Diff line change
Expand Up @@ -90,9 +90,15 @@ class Location:
def __init__(self, scheme: str, path: str, storage_options: dict | None = None):
self.scheme = scheme
self.path = path
self.url = f"{self.scheme}://{self.path}"
self.storage_options = storage_options

@property
def url(self):
if self.path.startswith(self.scheme):
return self.path
else:
return f"{self.scheme}://{self.path}"

@classmethod
def with_location(cls, loc: str | Any, storage_options: dict | None = None) -> "Location":
"""Create a Location from a string or Location.
Expand Down

0 comments on commit 3f805fe

Please sign in to comment.