Skip to content

Commit

Permalink
Return url for file in GCP public bucket instead of error
Browse files Browse the repository at this point in the history
  • Loading branch information
dreadatour committed Dec 27, 2024
1 parent cf05881 commit 035a2fc
Show file tree
Hide file tree
Showing 2 changed files with 16 additions and 19 deletions.
22 changes: 15 additions & 7 deletions src/datachain/client/gcs.py
Original file line number Diff line number Diff line change
Expand Up @@ -33,13 +33,21 @@ def create_fs(cls, **kwargs) -> GCSFileSystem:
return cast(GCSFileSystem, super().create_fs(**kwargs))

def url(self, path: str, expires: int = 3600, **kwargs) -> str:
try:
return self.fs.sign(self.get_full_path(path), expiration=expires, **kwargs)
except AttributeError as exc:
is_anon = self.fs.storage_options.get("token") == "anon"
if is_anon and "you need a private key to sign credentials" in str(exc):
return f"https://storage.googleapis.com/{self.name}/{path}"
raise
"""
Generate a signed URL for the given path.
If the client is anonymous, a public URL is returned instead.
"""
fs = cast(GCSFileSystem, self.fs)
if fs.storage_options.get("token") == "anon":
return self.public_url(path)
return fs.sign(self.get_full_path(path), expiration=expires, **kwargs)

Check warning on line 43 in src/datachain/client/gcs.py

View check run for this annotation

Codecov / codecov/patch

src/datachain/client/gcs.py#L43

Added line #L43 was not covered by tests

def public_url(self, path: str) -> str:
"""
Generate a public URL for the given path.
See https://cloud.google.com/storage/docs/access-public-data#api-link
"""
return f"https://storage.googleapis.com/{self.name}/{path}"

@staticmethod
def parse_timestamp(timestamp: str) -> datetime:
Expand Down
13 changes: 1 addition & 12 deletions tests/unit/test_client_gcs.py
Original file line number Diff line number Diff line change
@@ -1,17 +1,6 @@
from datachain.client import Client


def test_anon_url(mocker):
def sign(*args, **kwargs):
raise AttributeError(
"you need a private key to sign credentials."
"the credentials you are currently using"
" <class 'google.oauth2.credentials.Credentials'> just contains a token."
" see https://googleapis.dev/python/google-api-core/latest/auth.html"
"#setting-up-a-service-account for more details."
)

mocker.patch("gcsfs.GCSFileSystem.sign", side_effect=sign)

def test_anon_url():
client = Client.get_client("gs://foo", None, anon=True)
assert client.url("bar") == "https://storage.googleapis.com/foo/bar"

0 comments on commit 035a2fc

Please sign in to comment.