Skip to content

Commit

Permalink
Fix import and add commeent
Browse files Browse the repository at this point in the history
  • Loading branch information
maddiedawson committed Aug 4, 2023
1 parent e89be67 commit 988e827
Show file tree
Hide file tree
Showing 2 changed files with 11 additions and 2 deletions.
7 changes: 5 additions & 2 deletions streaming/base/storage/download.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,9 @@ def _create_s3_client(unsigned: bool = False) -> Any:
Args:
unsigned (bool, optional): Set to True if it is a public bucket.
Defaults to ``False``.
Returns:
botocore.client.BaseClient: The S3 client.
"""
if unsigned:
# Client will be using unsigned mode in which public
Expand Down Expand Up @@ -335,8 +338,8 @@ def download_from_databricks_unity_catalog(remote: str, local: str, timeout: flo
local (str): Local path (local filesystem).
timeout (float): Timeout in seconds.
"""
from util import (create_databricks_unity_catalog_s3_client,
get_databricks_unity_catalog_volume_info)
from streaming.base.storage.util import (create_databricks_unity_catalog_s3_client,
get_databricks_unity_catalog_volume_info)

obj = urllib.parse.urlparse(remote)
if obj.scheme != 'uc':
Expand Down
6 changes: 6 additions & 0 deletions streaming/base/storage/util.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,9 @@ def get_databricks_unity_catalog_volume_info(volume_name: str) -> Dict[str, Any]
Args:
volume_name (str): The name of the Databricks Unity Catalog volume.
Returns:
Dict[str, Any]: The volume info.
"""
import requests

Expand All @@ -32,6 +35,9 @@ def create_databricks_unity_catalog_s3_client(volume_id: str, timeout: float) ->
Args:
volume_id (str): The ID of the Databricks Unity Catalog volume.
timeout (float): The timeout for the S3 client.
Returns:
botocore.client.BaseClient: The S3 client.
"""
import boto3
from botocore.config import Config
Expand Down

0 comments on commit 988e827

Please sign in to comment.