Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[#5996] feat(python-client): Using credentail in python GVFS client. #5997

Merged
merged 25 commits into from
Jan 8, 2025
Merged
Show file tree
Hide file tree
Changes from 12 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ public long expireTimeInMs() {
public Map<String, String> credentialInfo() {
return (new ImmutableMap.Builder<String, String>())
.put(GRAVITINO_ADLS_SAS_TOKEN, sasToken)
.put(GRAVITINO_AZURE_STORAGE_ACCOUNT_NAME, accountName)
.build();
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -121,6 +121,7 @@ private IamPolicy createPolicy(
IamStatement.builder()
.effect(IamEffect.ALLOW)
.addAction("s3:GetObject")
.addAction("s3:GetObjectAttributes")
.addAction("s3:GetObjectVersion");
Map<String, IamStatement.Builder> bucketListStatmentBuilder = new HashMap<>();
Map<String, IamStatement.Builder> bucketGetLocationStatmentBuilder = new HashMap<>();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,12 @@ private CredentialAccessBoundary getAccessBoundary(
CredentialAccessBoundary.newBuilder();
readBuckets.forEach(
bucket -> {
AccessBoundaryRule rule1 =
AccessBoundaryRule.newBuilder()
.setAvailableResource(toGCSBucketResource(bucket))
.setAvailablePermissions(Arrays.asList("inRole:roles/storage.legacyBucketReader"))
.build();
credentialAccessBoundaryBuilder.addRule(rule1);
List<String> readConditions = readExpressions.get(bucket);
AccessBoundaryRule rule =
getAccessBoundaryRule(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,12 @@ class ADLSTokenCredential(Credential, ABC):

ADLS_TOKEN_CREDENTIAL_TYPE: str = "adls-token"
ADLS_DOMAIN: str = "dfs.core.windows.net"
_STORAGE_ACCOUNT_NAME: str = "azure-storage-account-name"
_SAS_TOKEN: str = "adls-sas-token"
STORAGE_ACCOUNT_NAME: str = "azure-storage-account-name"
FANNG1 marked this conversation as resolved.
Show resolved Hide resolved
SAS_TOKEN: str = "adls-sas-token"

def __init__(self, credential_info: Dict[str, str], expire_time_in_ms: int):
self._account_name = credential_info.get(self._STORAGE_ACCOUNT_NAME, None)
self._sas_token = credential_info.get(self._SAS_TOKEN, None)
self._account_name = credential_info.get(self.STORAGE_ACCOUNT_NAME, None)
self._sas_token = credential_info.get(self.SAS_TOKEN, None)
self._expire_time_in_ms = expire_time_in_ms
Precondition.check_string_not_empty(
self._account_name, "The ADLS account name should not be empty."
Expand Down Expand Up @@ -69,8 +69,8 @@ def credential_info(self) -> Dict[str, str]:
The credential information.
"""
return {
self._STORAGE_ACCOUNT_NAME: self._account_name,
self._SAS_TOKEN: self._sas_token,
self.STORAGE_ACCOUNT_NAME: self._account_name,
self.SAS_TOKEN: self._sas_token,
}

def account_name(self) -> str:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,12 @@ class AzureAccountKeyCredential(Credential, ABC):
"""Represents Azure account key credential."""

AZURE_ACCOUNT_KEY_CREDENTIAL_TYPE: str = "azure-account-key"
_STORAGE_ACCOUNT_NAME: str = "azure-storage-account-name"
_STORAGE_ACCOUNT_KEY: str = "azure-storage-account-key"
STORAGE_ACCOUNT_NAME: str = "azure-storage-account-name"
STORAGE_ACCOUNT_KEY: str = "azure-storage-account-key"

def __init__(self, credential_info: Dict[str, str], expire_time_in_ms: int):
self._account_name = credential_info.get(self._STORAGE_ACCOUNT_NAME, None)
self._account_key = credential_info.get(self._STORAGE_ACCOUNT_KEY, None)
self._account_name = credential_info.get(self.STORAGE_ACCOUNT_NAME, None)
self._account_key = credential_info.get(self.STORAGE_ACCOUNT_KEY, None)
Precondition.check_string_not_empty(
self._account_name, "The Azure account name should not be empty"
)
Expand Down Expand Up @@ -67,8 +67,8 @@ def credential_info(self) -> Dict[str, str]:
The credential information.
"""
return {
self._STORAGE_ACCOUNT_NAME: self._account_name,
self._STORAGE_ACCOUNT_KEY: self._account_key,
self.STORAGE_ACCOUNT_NAME: self._account_name,
self.STORAGE_ACCOUNT_KEY: self._account_key,
}

def account_name(self) -> str:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,12 +26,12 @@ class GCSTokenCredential(Credential, ABC):
"""Represents the GCS token credential."""

GCS_TOKEN_CREDENTIAL_TYPE: str = "gcs-token"
_GCS_TOKEN_NAME: str = "token"
GCS_TOKEN_NAME: str = "token"

_expire_time_in_ms: int = 0

def __init__(self, credential_info: Dict[str, str], expire_time_in_ms: int):
self._token = credential_info.get(self._GCS_TOKEN_NAME, None)
self._token = credential_info.get(self.GCS_TOKEN_NAME, None)
self._expire_time_in_ms = expire_time_in_ms
Precondition.check_string_not_empty(
self._token, "GCS token should not be empty"
Expand Down Expand Up @@ -64,7 +64,7 @@ def credential_info(self) -> Dict[str, str]:
Returns:
The credential information.
"""
return {self._GCS_TOKEN_NAME: self._token}
return {self.GCS_TOKEN_NAME: self._token}

def token(self) -> str:
"""The GCS token.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,13 +26,13 @@ class OSSSecretKeyCredential(Credential, ABC):
"""Represents OSS secret key credential."""

OSS_SECRET_KEY_CREDENTIAL_TYPE: str = "oss-secret-key"
_STATIC_ACCESS_KEY_ID: str = "oss-access-key-id"
_STATIC_SECRET_ACCESS_KEY: str = "oss-secret-access-key"
STATIC_ACCESS_KEY_ID: str = "oss-access-key-id"
STATIC_SECRET_ACCESS_KEY: str = "oss-secret-access-key"

def __init__(self, credential_info: Dict[str, str], expire_time_in_ms: int):
self._access_key_id = credential_info.get(self._STATIC_ACCESS_KEY_ID, None)
self._access_key_id = credential_info.get(self.STATIC_ACCESS_KEY_ID, None)
self._secret_access_key = credential_info.get(
self._STATIC_SECRET_ACCESS_KEY, None
self.STATIC_SECRET_ACCESS_KEY, None
)
Precondition.check_string_not_empty(
self._access_key_id, "The OSS access key ID should not be empty"
Expand Down Expand Up @@ -69,8 +69,8 @@ def credential_info(self) -> Dict[str, str]:
The credential information.
"""
return {
self._STATIC_ACCESS_KEY_ID: self._access_key_id,
self._STATIC_SECRET_ACCESS_KEY: self._secret_access_key,
self.STATIC_ACCESS_KEY_ID: self._access_key_id,
self.STATIC_SECRET_ACCESS_KEY: self._secret_access_key,
}

def access_key_id(self) -> str:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,16 +26,16 @@ class OSSTokenCredential(Credential, ABC):
"""Represents OSS token credential."""

OSS_TOKEN_CREDENTIAL_TYPE: str = "oss-token"
_STATIC_ACCESS_KEY_ID: str = "oss-access-key-id"
_STATIC_SECRET_ACCESS_KEY: str = "oss-secret-access-key"
_OSS_TOKEN: str = "oss-security-token"
STATIC_ACCESS_KEY_ID: str = "oss-access-key-id"
STATIC_SECRET_ACCESS_KEY: str = "oss-secret-access-key"
OSS_TOKEN: str = "oss-security-token"

def __init__(self, credential_info: Dict[str, str], expire_time_in_ms: int):
self._access_key_id = credential_info.get(self._STATIC_ACCESS_KEY_ID, None)
self._access_key_id = credential_info.get(self.STATIC_ACCESS_KEY_ID, None)
self._secret_access_key = credential_info.get(
self._STATIC_SECRET_ACCESS_KEY, None
self.STATIC_SECRET_ACCESS_KEY, None
)
self._security_token = credential_info.get(self._OSS_TOKEN, None)
self._security_token = credential_info.get(self.OSS_TOKEN, None)
self._expire_time_in_ms = expire_time_in_ms
Precondition.check_string_not_empty(
self._access_key_id, "The OSS access key ID should not be empty"
Expand Down Expand Up @@ -75,9 +75,9 @@ def credential_info(self) -> Dict[str, str]:
The credential information.
"""
return {
self._STATIC_ACCESS_KEY_ID: self._access_key_id,
self._STATIC_SECRET_ACCESS_KEY: self._secret_access_key,
self._OSS_TOKEN: self._security_token,
self.STATIC_ACCESS_KEY_ID: self._access_key_id,
self.STATIC_SECRET_ACCESS_KEY: self._secret_access_key,
self.OSS_TOKEN: self._security_token,
}

def access_key_id(self) -> str:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,13 +26,13 @@ class S3SecretKeyCredential(Credential, ABC):
"""Represents S3 secret key credential."""

S3_SECRET_KEY_CREDENTIAL_TYPE: str = "s3-secret-key"
_STATIC_ACCESS_KEY_ID: str = "s3-access-key-id"
_STATIC_SECRET_ACCESS_KEY: str = "s3-secret-access-key"
STATIC_ACCESS_KEY_ID: str = "s3-access-key-id"
STATIC_SECRET_ACCESS_KEY: str = "s3-secret-access-key"

def __init__(self, credential_info: Dict[str, str], expire_time: int):
self._access_key_id = credential_info.get(self._STATIC_ACCESS_KEY_ID, None)
self._access_key_id = credential_info.get(self.STATIC_ACCESS_KEY_ID, None)
self._secret_access_key = credential_info.get(
self._STATIC_SECRET_ACCESS_KEY, None
self.STATIC_SECRET_ACCESS_KEY, None
)
Precondition.check_string_not_empty(
self._access_key_id, "S3 access key id should not be empty"
Expand Down Expand Up @@ -70,8 +70,8 @@ def credential_info(self) -> Dict[str, str]:
The credential information.
"""
return {
self._STATIC_ACCESS_KEY_ID: self._access_key_id,
self._STATIC_SECRET_ACCESS_KEY: self._secret_access_key,
self.STATIC_ACCESS_KEY_ID: self._access_key_id,
self.STATIC_SECRET_ACCESS_KEY: self._secret_access_key,
}

def access_key_id(self) -> str:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,21 +26,21 @@ class S3TokenCredential(Credential, ABC):
"""Represents the S3 token credential."""

S3_TOKEN_CREDENTIAL_TYPE: str = "s3-token"
_SESSION_ACCESS_KEY_ID: str = "s3-access-key-id"
_SESSION_SECRET_ACCESS_KEY: str = "s3-secret-access-key"
_SESSION_TOKEN: str = "s3-session-token"
SESSION_ACCESS_KEY_ID: str = "s3-access-key-id"
SESSION_SECRET_ACCESS_KEY: str = "s3-secret-access-key"
SESSION_TOKEN: str = "s3-session-token"

_expire_time_in_ms: int = 0
_access_key_id: str = None
_secret_access_key: str = None
_session_token: str = None

def __init__(self, credential_info: Dict[str, str], expire_time_in_ms: int):
self._access_key_id = credential_info.get(self._SESSION_ACCESS_KEY_ID, None)
self._access_key_id = credential_info.get(self.SESSION_ACCESS_KEY_ID, None)
self._secret_access_key = credential_info.get(
self._SESSION_SECRET_ACCESS_KEY, None
self.SESSION_SECRET_ACCESS_KEY, None
)
self._session_token = credential_info.get(self._SESSION_TOKEN, None)
self._session_token = credential_info.get(self.SESSION_TOKEN, None)
self._expire_time_in_ms = expire_time_in_ms
Precondition.check_string_not_empty(
self._access_key_id, "The S3 access key ID should not be empty"
Expand Down Expand Up @@ -80,9 +80,9 @@ def credential_info(self) -> Dict[str, str]:
The credential information.
"""
return {
self._SESSION_ACCESS_KEY_ID: self._access_key_id,
self._SESSION_SECRET_ACCESS_KEY: self._secret_access_key,
self._SESSION_TOKEN: self._session_token,
self.SESSION_ACCESS_KEY_ID: self._access_key_id,
self.SESSION_SECRET_ACCESS_KEY: self._secret_access_key,
self.SESSION_TOKEN: self._session_token,
}

def access_key_id(self) -> str:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ def __init__(
metadata_object_type = metadata_object.type().value
metadata_object_name = metadata_object.name()
self._request_path = (
f"api/metalakes/{metalake_name}objects/{metadata_object_type}/"
f"api/metalakes/{metalake_name}/objects/{metadata_object_type}/"
f"{metadata_object_name}/credentials"
)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ def validate(self):

for alias in self._aliases or []:
if not self._is_not_blank(alias):
raise IllegalArgumentException('Alias must not be null or empty')
raise IllegalArgumentException("Alias must not be null or empty")

def _is_not_blank(self, string: str) -> bool:
return string is not None and string.strip()
Loading
Loading