Skip to content

Commit 634d28b

Browse files
anjaliratnam-msftanjaliratnam-msft
committed
updates
1 parent f64dff2 commit 634d28b

File tree

6 files changed

+134
-132
lines changed

6 files changed

+134
-132
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
Unreleased
44
----------
55

6-
- .
6+
- Add adlfs user agent
77

88

99
2024.12.0

‎adlfs/__init__.py‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,5 @@
11
from .gen1 import AzureDatalakeFileSystem
22
from .spec import AzureBlobFile, AzureBlobFileSystem
3+
from .utils import __version__, version_tuple # noqa: F401
34

45
__all__ = ["AzureBlobFileSystem", "AzureBlobFile", "AzureDatalakeFileSystem"]
5-
6-
try:
7-
from ._version import version as __version__ # type: ignore[import]
8-
from ._version import version_tuple # type: ignore[import]
9-
except ImportError:
10-
__version__ = "UNKNOWN"
11-
version_tuple = (0, 0, __version__) # type: ignore[assignment]

‎adlfs/spec.py‎

Lines changed: 18 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
from fsspec.utils import infer_storage_options
3939

4040
from .utils import (
41+
__version__,
4142
close_container_client,
4243
close_credential,
4344
close_service_client,
@@ -72,11 +73,7 @@
7273

7374
_SOCKET_TIMEOUT_DEFAULT = object()
7475

75-
76-
def _get_user_agent():
77-
from adlfs import __version__
78-
79-
return f"adlfs/{__version__}"
76+
_USER_AGENT = f"adlfs/{__version__}"
8077

8178

8279
# https://github.com/Azure/azure-sdk-for-python/issues/11419#issuecomment-628143480
@@ -129,33 +126,23 @@ def _create_aio_blob_service_client(
129126
location_mode: Optional[str] = None,
130127
credential: Optional[str] = None,
131128
) -> AIOBlobServiceClient:
129+
service_client_kwargs = {
130+
"account_url": account_url,
131+
"user_agent": _USER_AGENT,
132+
}
132133
if credential is not None:
133-
return AIOBlobServiceClient(
134-
account_url=account_url,
135-
credential=credential,
136-
_location_mode=location_mode,
137-
user_agent=_get_user_agent(),
138-
)
134+
service_client_kwargs["credential"] = credential
139135
elif location_mode is not None:
140-
return AIOBlobServiceClient(
141-
account_url=account_url,
142-
credential=None,
143-
_location_mode=location_mode,
144-
user_agent=_get_user_agent(),
145-
)
146-
else:
147-
return AIOBlobServiceClient(
148-
account_url=account_url,
149-
user_agent=_get_user_agent(),
150-
)
136+
service_client_kwargs["_location_mode"] = location_mode
137+
return AIOBlobServiceClient(**service_client_kwargs)
151138

152139

153140
def _create_aio_blob_service_client_from_connection_string(
154141
connection_string: str,
155142
) -> AIOBlobServiceClient:
156143
return AIOBlobServiceClient.from_connection_string(
157144
conn_str=connection_string,
158-
user_agent=_get_user_agent(),
145+
user_agent=_USER_AGENT,
159146
)
160147

161148

@@ -521,7 +508,10 @@ def do_connect(self):
521508
)
522509
elif self.account_name is not None:
523510
if hasattr(self, "account_host"):
524-
self.account_url: str = f"https://{self.account_host}"
511+
if self.account_host.startswith("http://"):
512+
self.account_url: str = self.account_host
513+
else:
514+
self.account_url: str = f"https://{self.account_host}"
525515
else:
526516
self.account_url: str = (
527517
f"https://{self.account_name}.blob.core.windows.net"
@@ -2080,7 +2070,10 @@ def connect_client(self):
20802070
"""
20812071
try:
20822072
if hasattr(self.fs, "account_host"):
2083-
self.fs.account_url: str = f"https://{self.fs.account_host}"
2073+
if self.fs.account_host.startswith("http://"):
2074+
self.account_url: str = self.fs.account_host
2075+
else:
2076+
self.fs.account_url: str = f"https://{self.fs.account_host}"
20842077
else:
20852078
self.fs.account_url: str = (
20862079
f"https://{self.fs.account_name}.blob.core.windows.net"

‎adlfs/tests/test_spec.py‎

Lines changed: 0 additions & 99 deletions
Original file line numberDiff line numberDiff line change
@@ -2,15 +2,13 @@
22
import os
33
import tempfile
44
from unittest import mock
5-
from unittest.mock import patch
65

76
import azure.storage.blob.aio
87
import dask.dataframe as dd
98
import fsspec
109
import numpy as np
1110
import pandas as pd
1211
import pytest
13-
from azure.storage.blob.aio import BlobServiceClient as AIOBlobServiceClient
1412
from packaging.version import parse as parse_version
1513
from pandas.testing import assert_frame_equal
1614

@@ -2047,100 +2045,3 @@ def test_open_file_x(storage: azure.storage.blob.BlobServiceClient, tmpdir):
20472045
with fs.open("data/afile", "xb") as f:
20482046
pass
20492047
assert fs.cat_file("data/afile") == b"data"
2050-
2051-
2052-
def test_user_agent_blob_file_connection_str(
2053-
storage: azure.storage.blob.BlobServiceClient, mocker
2054-
):
2055-
from adlfs import __version__
2056-
2057-
fs = AzureBlobFileSystem(
2058-
account_name=storage.account_name, connection_string=CONN_STR
2059-
)
2060-
mock_client = mocker.patch.object(
2061-
AIOBlobServiceClient, "from_connection_string", return_value=mocker.MagicMock()
2062-
)
2063-
mock_client.return_value.get_container_client = mocker.MagicMock()
2064-
2065-
f = AzureBlobFile(fs, "data/root/a/file.txt", mode="rb")
2066-
f.container_name = "data"
2067-
f.connect_client()
2068-
2069-
mock_client.assert_called_once()
2070-
assert "user_agent" in mock_client.call_args.kwargs
2071-
assert mock_client.call_args.kwargs["user_agent"] == f"adlfs/{__version__}"
2072-
2073-
2074-
def test_user_agent_blob_file_initializer(
2075-
storage: azure.storage.blob.BlobServiceClient, mocker
2076-
):
2077-
from adlfs import __version__
2078-
2079-
path = "root/a/file.txt"
2080-
mocker.patch("adlfs.spec.filter_blobs", [])
2081-
2082-
mock_details = mocker.PropertyMock(
2083-
return_value={
2084-
"name": path,
2085-
"metadata": {},
2086-
"size": 0,
2087-
"content_settings": {"content_type": ""},
2088-
}
2089-
)
2090-
mocker.patch.object(AzureBlobFile, "details", mock_details)
2091-
mock_client = mocker.patch("adlfs.spec.AIOBlobServiceClient", spec=True)
2092-
2093-
mock_container_client = mocker.MagicMock()
2094-
mock_blob_client = mocker.MagicMock()
2095-
mock_blob_client.get_blob_properties = mocker.AsyncMock(return_value={})
2096-
mock_blob_client.__aenter__ = mocker.AsyncMock(return_value=mock_blob_client)
2097-
mock_blob_client.__aexit__ = mocker.AsyncMock()
2098-
mock_blob_client.close = mocker.AsyncMock()
2099-
mock_container_client.get_blob_client = mocker.MagicMock(
2100-
return_value=mock_blob_client
2101-
)
2102-
mock_client.return_value.get_container_client = mocker.MagicMock(
2103-
return_value=mock_container_client
2104-
)
2105-
2106-
fs = AzureBlobFileSystem(
2107-
account_name=storage.account_name,
2108-
)
2109-
AzureBlobFile(fs, "data/root/a/file.txt", mode="rb")
2110-
2111-
mock_client.assert_called_once()
2112-
assert "user_agent" in mock_client.call_args.kwargs
2113-
assert mock_client.call_args.kwargs["user_agent"] == f"adlfs/{__version__}"
2114-
2115-
2116-
@patch("adlfs.spec.AIOBlobServiceClient")
2117-
def test_user_agent_connection_str(
2118-
storage: azure.storage.blob.BlobServiceClient, mocker
2119-
):
2120-
from adlfs import __version__
2121-
2122-
mock_client_instance = mocker.MagicMock()
2123-
mock_client_instance.close = mocker.AsyncMock()
2124-
mock_client = mocker.patch(
2125-
"adlfs.spec.AIOBlobServiceClient.from_connection_string",
2126-
return_value=mock_client_instance,
2127-
)
2128-
2129-
AzureBlobFileSystem(account_name=storage.account_name, connection_string=CONN_STR)
2130-
mock_client.assert_called_once()
2131-
assert "user_agent" in mock_client.call_args.kwargs
2132-
assert mock_client.call_args.kwargs["user_agent"] == f"adlfs/{__version__}"
2133-
2134-
2135-
def test_user_agent_initializer(storage: azure.storage.blob.BlobServiceClient, mocker):
2136-
from adlfs import __version__
2137-
2138-
fs = AzureBlobFileSystem(
2139-
account_name=storage.account_name,
2140-
)
2141-
mock_client = mocker.patch("adlfs.spec.AIOBlobServiceClient", spec=True)
2142-
2143-
fs.do_connect()
2144-
mock_client.assert_called_once()
2145-
assert "user_agent" in mock_client.call_args.kwargs
2146-
assert mock_client.call_args.kwargs["user_agent"] == f"adlfs/{__version__}"

‎adlfs/tests/test_user_agent.py‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,107 @@
1+
import datetime
2+
3+
import azure.storage.blob
4+
import pytest
5+
from azure.storage.blob import BlobServiceClient, PublicAccess
6+
from azure.storage.blob.aio import BlobServiceClient as AIOBlobServiceClient
7+
8+
from adlfs import AzureBlobFile, AzureBlobFileSystem
9+
from adlfs.utils import __version__ as __version__
10+
11+
URL = "http://127.0.0.1:10000"
12+
ACCOUNT_NAME = "devstoreaccount1"
13+
KEY = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==" # NOQA
14+
CONN_STR = f"DefaultEndpointsProtocol=http;AccountName={ACCOUNT_NAME};AccountKey={KEY};BlobEndpoint={URL}/{ACCOUNT_NAME};" # NOQA
15+
DEFAULT_VERSION_ID = "1970-01-01T00:00:00.0000000Z"
16+
LATEST_VERSION_ID = "2022-01-01T00:00:00.0000000Z"
17+
18+
_USER_AGENT = f"adlfs/{__version__}"
19+
20+
21+
def assert_sets_adlfs_user_agent(mock_client):
22+
mock_client.assert_called_once()
23+
assert "user_agent" in mock_client.call_args.kwargs
24+
assert mock_client.call_args.kwargs["user_agent"] == _USER_AGENT
25+
26+
27+
@pytest.fixture()
28+
def mock_from_connection_string(mocker):
29+
return mocker.patch.object(
30+
AIOBlobServiceClient,
31+
"from_connection_string",
32+
autospec=True,
33+
side_effect=AIOBlobServiceClient.from_connection_string,
34+
)
35+
36+
37+
@pytest.fixture()
38+
def mock_service_client_init(mocker):
39+
return mocker.patch.object(
40+
AIOBlobServiceClient,
41+
"__init__",
42+
autospec=True,
43+
side_effect=AIOBlobServiceClient.__init__,
44+
)
45+
46+
47+
@pytest.fixture(scope="function")
48+
def mock_container(host):
49+
conn_str = f"DefaultEndpointsProtocol=http;AccountName={ACCOUNT_NAME};AccountKey={KEY};BlobEndpoint={URL}/{ACCOUNT_NAME};" # NOQA
50+
51+
bbs = BlobServiceClient.from_connection_string(conn_str=conn_str)
52+
if "data2" not in [c["name"] for c in bbs.list_containers()]:
53+
bbs.create_container("data2", public_access=PublicAccess.Container)
54+
container_client = bbs.get_container_client(container="data2")
55+
bbs.insert_time = datetime.datetime.now(tz=datetime.timezone.utc).replace(
56+
microsecond=0
57+
)
58+
container_client.upload_blob("root/a/file.txt", b"0123456789")
59+
60+
yield bbs
61+
62+
bbs.delete_container("data2")
63+
64+
65+
def test_user_agent_blob_file_connection_str(
66+
storage: azure.storage.blob.BlobServiceClient, mock_from_connection_string
67+
):
68+
fs = AzureBlobFileSystem(
69+
account_name=storage.account_name,
70+
connection_string=CONN_STR,
71+
skip_instance_cache=True,
72+
)
73+
AzureBlobFile(fs, "data/root/a/file.txt", mode="rb")
74+
75+
assert_sets_adlfs_user_agent(mock_from_connection_string)
76+
77+
78+
def test_user_agent_blob_file_initializer(mock_container, mock_service_client_init):
79+
fs = AzureBlobFileSystem(
80+
account_name=ACCOUNT_NAME,
81+
account_host="http://127.0.0.1:10000/devstoreaccount1",
82+
account_key=KEY,
83+
skip_instance_cache=True,
84+
)
85+
AzureBlobFile(fs, "data2/root/a/file.txt", mode="rb")
86+
assert_sets_adlfs_user_agent(mock_service_client_init)
87+
88+
89+
def test_user_agent_connection_str(
90+
storage: azure.storage.blob.BlobServiceClient, mock_from_connection_string
91+
):
92+
AzureBlobFileSystem(
93+
account_name=storage.account_name,
94+
connection_string=CONN_STR,
95+
skip_instance_cache=True,
96+
)
97+
assert_sets_adlfs_user_agent(mock_from_connection_string)
98+
99+
100+
def test_user_agent_initializer(
101+
storage: azure.storage.blob.BlobServiceClient, mock_service_client_init
102+
):
103+
AzureBlobFileSystem(
104+
account_name=storage.account_name,
105+
skip_instance_cache=True,
106+
)
107+
assert_sets_adlfs_user_agent(mock_service_client_init)

‎adlfs/utils.py‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,12 @@
11
from typing import Optional
22

3+
try:
4+
from ._version import version as __version__ # type: ignore[import]
5+
from ._version import version_tuple # type: ignore[import]
6+
except ImportError:
7+
__version__ = "UNKNOWN"
8+
version_tuple = (0, 0, __version__) # type: ignore[assignment]
9+
310

411
def match_blob_version(blob, version_id: Optional[str]):
512
blob_version_id = blob.get("version_id")

0 commit comments

Comments
 (0)