From 449ae9ad2d621093a3b207664a842d97a87a347d Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Tue, 8 Jul 2025 17:06:38 +0200 Subject: [PATCH 1/3] Python client: add support for endpoint, sts-endpoint, path-style-access Amends #1913 and #2012 --- client/python/cli/command/__init__.py | 3 +++ client/python/cli/command/catalogs.py | 6 ++++++ client/python/cli/constants.py | 6 ++++++ client/python/cli/options/option_tree.py | 3 +++ 4 files changed, 18 insertions(+) diff --git a/client/python/cli/command/__init__.py b/client/python/cli/command/__init__.py index 46fe2d49b8..659a9b9e2c 100644 --- a/client/python/cli/command/__init__.py +++ b/client/python/cli/command/__init__.py @@ -65,6 +65,9 @@ def options_get(key, f=lambda x: x): hadoop_warehouse=options_get(Arguments.HADOOP_WAREHOUSE), iceberg_remote_catalog_name=options_get(Arguments.ICEBERG_REMOTE_CATALOG_NAME), remove_properties=[] if remove_properties is None else remove_properties, + endpoint=options_get(Arguments.ENDPOINT), + sts_endpoint=options_get(Arguments.STS_ENDPOINT), + path_style_access=options_get(Arguments.PATH_STYLE_ACCESS), catalog_connection_type=options_get(Arguments.CATALOG_CONNECTION_TYPE), catalog_authentication_type=options_get(Arguments.CATALOG_AUTHENTICATION_TYPE), catalog_service_identity_type=options_get(Arguments.CATALOG_SERVICE_IDENTITY_TYPE), diff --git a/client/python/cli/command/catalogs.py b/client/python/cli/command/catalogs.py index 501e9eefbd..cf3e383972 100644 --- a/client/python/cli/command/catalogs.py +++ b/client/python/cli/command/catalogs.py @@ -64,6 +64,9 @@ class CatalogsCommand(Command): remove_properties: List[str] hadoop_warehouse: str iceberg_remote_catalog_name: str + endpoint: str + sts_endpoint: str + path_style_access: bool catalog_connection_type: str catalog_authentication_type: str catalog_service_identity_type: str @@ -175,6 +178,9 @@ def _build_storage_config_info(self): external_id=self.external_id, user_arn=self.user_arn, region=self.region, + endpoint=self.endpoint, + sts_endpoint=self.sts_endpoint, + path_style_access=self.path_style_access, ) elif self.storage_type == StorageType.AZURE.value: config = AzureStorageConfigInfo( diff --git a/client/python/cli/constants.py b/client/python/cli/constants.py index 93b36d998c..26ba80362b 100644 --- a/client/python/cli/constants.py +++ b/client/python/cli/constants.py @@ -165,6 +165,9 @@ class Arguments: PROXY = "proxy" HADOOP_WAREHOUSE = "hadoop_warehouse" ICEBERG_REMOTE_CATALOG_NAME = "iceberg_remote_catalog_name" + ENDPOINT = 'endpoint' + STS_ENDPOINT = 'sts_endpoint' + PATH_STYLE_ACCESS = 'path_style_access' CATALOG_CONNECTION_TYPE = "catalog_connection_type" CATALOG_AUTHENTICATION_TYPE = "catalog_authentication_type" CATALOG_SERVICE_IDENTITY_TYPE = "catalog_service_identity_type" @@ -222,6 +225,9 @@ class Create: EXTERNAL_ID = "(Only for S3) The external ID to use when connecting to S3" REGION = "(Only for S3) The region to use when connecting to S3" USER_ARN = "(Only for S3) A user ARN to use when connecting to S3" + ENDPOINT = "(Only for S3) The S3 endpoint to use when connecting to S3" + STS_ENDPOINT = "(Only for S3) The STS endpoint to use when connecting to STS" + PATH_STYLE_ACCESS = "(Only for S3) whether to use path-style-access for S3" TENANT_ID = "(Required for Azure) A tenant ID to use when connecting to Azure Storage" MULTI_TENANT_APP_NAME = ( diff --git a/client/python/cli/options/option_tree.py b/client/python/cli/options/option_tree.py index d8db86899d..7b10a64ea6 100644 --- a/client/python/cli/options/option_tree.py +++ b/client/python/cli/options/option_tree.py @@ -116,6 +116,9 @@ def get_tree() -> List[Option]: Argument(Arguments.STORAGE_TYPE, str, Hints.Catalogs.Create.STORAGE_TYPE, lower=True, choices=[st.value for st in StorageType]), Argument(Arguments.DEFAULT_BASE_LOCATION, str, Hints.Catalogs.Create.DEFAULT_BASE_LOCATION), + Argument(Arguments.ENDPOINT, str, Hints.Catalogs.Create.ENDPOINT), + Argument(Arguments.STS_ENDPOINT, str, Hints.Catalogs.Create.STS_ENDPOINT), + Argument(Arguments.PATH_STYLE_ACCESS, bool, Hints.Catalogs.Create.PATH_STYLE_ACCESS), Argument(Arguments.ALLOWED_LOCATION, str, Hints.Catalogs.Create.ALLOWED_LOCATION, allow_repeats=True), Argument(Arguments.ROLE_ARN, str, Hints.Catalogs.Create.ROLE_ARN), From 694d57721315c8bbe8b0100236ba0b5343944091 Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Thu, 17 Jul 2025 15:21:25 +0200 Subject: [PATCH 2/3] ruff --- client/python/cli/constants.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/client/python/cli/constants.py b/client/python/cli/constants.py index 26ba80362b..ea3056ae59 100644 --- a/client/python/cli/constants.py +++ b/client/python/cli/constants.py @@ -165,9 +165,9 @@ class Arguments: PROXY = "proxy" HADOOP_WAREHOUSE = "hadoop_warehouse" ICEBERG_REMOTE_CATALOG_NAME = "iceberg_remote_catalog_name" - ENDPOINT = 'endpoint' - STS_ENDPOINT = 'sts_endpoint' - PATH_STYLE_ACCESS = 'path_style_access' + ENDPOINT = "endpoint" + STS_ENDPOINT = "sts_endpoint" + PATH_STYLE_ACCESS = "path_style_access" CATALOG_CONNECTION_TYPE = "catalog_connection_type" CATALOG_AUTHENTICATION_TYPE = "catalog_authentication_type" CATALOG_SERVICE_IDENTITY_TYPE = "catalog_service_identity_type" @@ -226,7 +226,9 @@ class Create: REGION = "(Only for S3) The region to use when connecting to S3" USER_ARN = "(Only for S3) A user ARN to use when connecting to S3" ENDPOINT = "(Only for S3) The S3 endpoint to use when connecting to S3" - STS_ENDPOINT = "(Only for S3) The STS endpoint to use when connecting to STS" + STS_ENDPOINT = ( + "(Only for S3) The STS endpoint to use when connecting to STS" + ) PATH_STYLE_ACCESS = "(Only for S3) whether to use path-style-access for S3" TENANT_ID = "(Required for Azure) A tenant ID to use when connecting to Azure Storage" From 934f32a5162d0617cb64c043b43f3dd6dcea3100 Mon Sep 17 00:00:00 2001 From: Robert Stupp Date: Fri, 18 Jul 2025 08:04:12 +0200 Subject: [PATCH 3/3] review --- client/python/cli/command/catalogs.py | 2 +- client/python/cli/constants.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/client/python/cli/command/catalogs.py b/client/python/cli/command/catalogs.py index cf3e383972..034bd9b2e0 100644 --- a/client/python/cli/command/catalogs.py +++ b/client/python/cli/command/catalogs.py @@ -160,7 +160,7 @@ def validate(self): ) def _has_aws_storage_info(self): - return self.role_arn or self.external_id or self.user_arn or self.region + return self.role_arn or self.external_id or self.user_arn or self.region or self.endpoint or self.sts_endpoint or self.path_style_access def _has_azure_storage_info(self): return self.tenant_id or self.multi_tenant_app_name or self.consent_url diff --git a/client/python/cli/constants.py b/client/python/cli/constants.py index ea3056ae59..19f132673b 100644 --- a/client/python/cli/constants.py +++ b/client/python/cli/constants.py @@ -229,7 +229,7 @@ class Create: STS_ENDPOINT = ( "(Only for S3) The STS endpoint to use when connecting to STS" ) - PATH_STYLE_ACCESS = "(Only for S3) whether to use path-style-access for S3" + PATH_STYLE_ACCESS = "(Only for S3) Whether to use path-style-access for S3" TENANT_ID = "(Required for Azure) A tenant ID to use when connecting to Azure Storage" MULTI_TENANT_APP_NAME = (