diff --git a/client/python/cli/command/__init__.py b/client/python/cli/command/__init__.py index 897cbaaa52..46fe2d49b8 100644 --- a/client/python/cli/command/__init__.py +++ b/client/python/cli/command/__init__.py @@ -39,6 +39,7 @@ def options_get(key, f=lambda x: x): properties = Parser.parse_properties(options_get(Arguments.PROPERTY)) set_properties = Parser.parse_properties(options_get(Arguments.SET_PROPERTY)) remove_properties = options_get(Arguments.REMOVE_PROPERTY) + catalog_client_scopes = options_get(Arguments.CATALOG_CLIENT_SCOPE) command = None if options.command == Commands.CATALOGS: @@ -61,9 +62,24 @@ def options_get(key, f=lambda x: x): catalog_name=options_get(Arguments.CATALOG), properties={} if properties is None else properties, set_properties={} if set_properties is None else set_properties, - remove_properties=[] - if remove_properties is None - else remove_properties, + hadoop_warehouse=options_get(Arguments.HADOOP_WAREHOUSE), + iceberg_remote_catalog_name=options_get(Arguments.ICEBERG_REMOTE_CATALOG_NAME), + remove_properties=[] if remove_properties is None else remove_properties, + catalog_connection_type=options_get(Arguments.CATALOG_CONNECTION_TYPE), + catalog_authentication_type=options_get(Arguments.CATALOG_AUTHENTICATION_TYPE), + catalog_service_identity_type=options_get(Arguments.CATALOG_SERVICE_IDENTITY_TYPE), + catalog_service_identity_iam_arn=options_get(Arguments.CATALOG_SERVICE_IDENTITY_IAM_ARN), + catalog_uri=options_get(Arguments.CATALOG_URI), + catalog_token_uri=options_get(Arguments.CATALOG_TOKEN_URI), + catalog_client_id=options_get(Arguments.CATALOG_CLIENT_ID), + catalog_client_secret=options_get(Arguments.CATALOG_CLIENT_SECRET), + catalog_client_scopes=[] if catalog_client_scopes is None else catalog_client_scopes, + catalog_bearer_token=options_get(Arguments.CATALOG_BEARER_TOKEN), + catalog_role_arn=options_get(Arguments.CATALOG_ROLE_ARN), + catalog_role_session_name=options_get(Arguments.CATALOG_ROLE_SESSION_NAME), + catalog_external_id=options_get(Arguments.CATALOG_EXTERNAL_ID), + catalog_signing_region=options_get(Arguments.CATALOG_SIGNING_REGION), + catalog_signing_name=options_get(Arguments.CATALOG_SIGNING_NAME) ) elif options.command == Commands.PRINCIPALS: from cli.command.principals import PrincipalsCommand diff --git a/client/python/cli/command/catalogs.py b/client/python/cli/command/catalogs.py index 0baf2cffad..501e9eefbd 100644 --- a/client/python/cli/command/catalogs.py +++ b/client/python/cli/command/catalogs.py @@ -19,23 +19,17 @@ from dataclasses import dataclass from typing import Dict, List -from pydantic import StrictStr +from pydantic import StrictStr, SecretStr from cli.command import Command -from cli.constants import StorageType, CatalogType, Subcommands, Arguments +from cli.constants import StorageType, CatalogType, CatalogConnectionType, Subcommands, Arguments, AuthenticationType, \ + ServiceIdentityType from cli.options.option_tree import Argument -from polaris.management import ( - PolarisDefaultApi, - CreateCatalogRequest, - UpdateCatalogRequest, - StorageConfigInfo, - ExternalCatalog, - AwsStorageConfigInfo, - AzureStorageConfigInfo, - GcpStorageConfigInfo, - PolarisCatalog, - CatalogProperties, -) +from polaris.management import PolarisDefaultApi, CreateCatalogRequest, UpdateCatalogRequest, \ + StorageConfigInfo, ExternalCatalog, AwsStorageConfigInfo, AzureStorageConfigInfo, GcpStorageConfigInfo, \ + PolarisCatalog, CatalogProperties, BearerAuthenticationParameters, \ + OAuthClientCredentialsParameters, SigV4AuthenticationParameters, HadoopConnectionConfigInfo, \ + IcebergRestConnectionConfigInfo, AwsIamServiceIdentityInfo @dataclass @@ -68,19 +62,56 @@ class CatalogsCommand(Command): properties: Dict[str, StrictStr] set_properties: Dict[str, StrictStr] remove_properties: List[str] + hadoop_warehouse: str + iceberg_remote_catalog_name: str + catalog_connection_type: str + catalog_authentication_type: str + catalog_service_identity_type: str + catalog_service_identity_iam_arn: str + catalog_uri: str + catalog_token_uri: str + catalog_client_id: str + catalog_client_secret: str + catalog_client_scopes: List[str] + catalog_bearer_token: str + catalog_role_arn: str + catalog_role_session_name: str + catalog_external_id: str + catalog_signing_region: str + catalog_signing_name: str def validate(self): if self.catalogs_subcommand == Subcommands.CREATE: - if not self.storage_type: - raise Exception( - f"Missing required argument:" - f" {Argument.to_flag_name(Arguments.STORAGE_TYPE)}" - ) - if not self.default_base_location: - raise Exception( - f"Missing required argument:" - f" {Argument.to_flag_name(Arguments.DEFAULT_BASE_LOCATION)}" - ) + if self.catalog_type != CatalogType.EXTERNAL.value: + if not self.storage_type: + raise Exception(f'Missing required argument:' + f' {Argument.to_flag_name(Arguments.STORAGE_TYPE)}') + if not self.default_base_location: + raise Exception(f'Missing required argument:' + f' {Argument.to_flag_name(Arguments.DEFAULT_BASE_LOCATION)}') + else: + if self.catalog_authentication_type == AuthenticationType.OAUTH.value: + if not self.catalog_token_uri or not self.catalog_client_id \ + or not self.catalog_client_secret or len(self.catalog_client_scopes) == 0: + raise Exception(f"Authentication type 'OAUTH' requires" + f" {Argument.to_flag_name(Arguments.CATALOG_TOKEN_URI)}," + f" {Argument.to_flag_name(Arguments.CATALOG_CLIENT_ID)}," + f" {Argument.to_flag_name(Arguments.CATALOG_CLIENT_SECRET)}," + f" and at least one {Argument.to_flag_name(Arguments.CATALOG_CLIENT_SCOPE)}.") + elif self.catalog_authentication_type == AuthenticationType.BEARER.value: + if not self.catalog_bearer_token: + raise Exception(f"Missing required argument for authentication type 'BEARER':" + f" {Argument.to_flag_name(Arguments.CATALOG_BEARER_TOKEN)}") + elif self.catalog_authentication_type == AuthenticationType.SIGV4.value: + if not self.catalog_role_arn or not self.catalog_signing_region: + raise Exception(f"Authentication type 'SIGV4 requires" + f" {Argument.to_flag_name(Arguments.CATALOG_ROLE_ARN)}" + f" and {Argument.to_flag_name(Arguments.CATALOG_SIGNING_REGION)}") + + if self.catalog_service_identity_type == ServiceIdentityType.AWS_IAM.value: + if not self.catalog_service_identity_iam_arn: + raise Exception(f"Missing required argument for service identity type 'AWS_IAM':" + f" {Argument.to_flag_name(Arguments.CATALOG_SERVICE_IDENTITY_IAM_ARN)}") if self.storage_type == StorageType.S3.value: if not self.role_arn: @@ -166,19 +197,81 @@ def _build_storage_config_info(self): ) return config + def _build_connection_config_info(self): + if self.catalog_type != CatalogType.EXTERNAL.value: + return None + + auth_params = None + if self.catalog_authentication_type == AuthenticationType.OAUTH.value: + auth_params = OAuthClientCredentialsParameters( + authentication_type=self.catalog_authentication_type.upper(), + token_uri=self.catalog_token_uri, + client_id=self.catalog_client_id, + client_secret=SecretStr(self.catalog_client_secret), + scopes=self.catalog_client_scopes + ) + elif self.catalog_authentication_type == AuthenticationType.BEARER.value: + auth_params = BearerAuthenticationParameters( + authentication_type=self.catalog_authentication_type.upper(), + bearer_token=SecretStr(self.catalog_bearer_token) + ) + elif self.catalog_authentication_type == AuthenticationType.SIGV4.value: + auth_params = SigV4AuthenticationParameters( + authentication_type=self.catalog_authentication_type.upper(), + role_arn=self.catalog_role_arn, + role_session_name=self.catalog_role_session_name, + external_id=self.catalog_external_id, + signing_region=self.catalog_signing_region, + signing_name=self.catalog_signing_name, + ) + elif self.catalog_authentication_type is not None: + raise Exception("Unknown authentication type:", self.catalog_authentication_type) + + service_identity = None + if self.catalog_service_identity_type == ServiceIdentityType.AWS_IAM: + service_identity = AwsIamServiceIdentityInfo( + identity_type=self.catalog_service_identity_type.upper(), + iam_arn=self.catalog_service_identity_iam_arn + ) + elif self.catalog_service_identity_type is not None: + raise Exception("Unknown service identity type:", self.catalog_service_identity_type) + + config = None + if self.catalog_connection_type == CatalogConnectionType.HADOOP.value: + config = HadoopConnectionConfigInfo( + connection_type=self.catalog_connection_type.upper(), + uri=self.catalog_uri, + authentication_parameters=auth_params, + service_identity=service_identity, + warehouse=self.hadoop_warehouse + ) + elif self.catalog_connection_type == CatalogConnectionType.ICEBERG.value: + config = IcebergRestConnectionConfigInfo( + connection_type=self.catalog_connection_type.upper().replace('-', '_'), + uri=self.catalog_uri, + authentication_parameters=auth_params, + service_identity=service_identity, + remote_catalog_name=self.iceberg_remote_catalog_name + ) + elif self.catalog_connection_type is not None: + raise Exception("Unknown catalog connection type:", self.catalog_connection_type) + return config + def execute(self, api: PolarisDefaultApi) -> None: if self.catalogs_subcommand == Subcommands.CREATE: - config = self._build_storage_config_info() + storage_config = self._build_storage_config_info() + connection_config = self._build_connection_config_info() if self.catalog_type == CatalogType.EXTERNAL.value: request = CreateCatalogRequest( catalog=ExternalCatalog( type=self.catalog_type.upper(), name=self.catalog_name, - storage_config_info=config, + storage_config_info=storage_config, properties=CatalogProperties( default_base_location=self.default_base_location, - additional_properties=self.properties, + additional_properties=self.properties ), + connection_config_info=connection_config ) ) else: @@ -186,11 +279,12 @@ def execute(self, api: PolarisDefaultApi) -> None: catalog=PolarisCatalog( type=self.catalog_type.upper(), name=self.catalog_name, - storage_config_info=config, + storage_config_info=storage_config, properties=CatalogProperties( default_base_location=self.default_base_location, - additional_properties=self.properties, + additional_properties=self.properties ), + connection_config_info=connection_config ) ) api.create_catalog(request) diff --git a/client/python/cli/command/namespaces.py b/client/python/cli/command/namespaces.py index 7631384731..3528a6ba12 100644 --- a/client/python/cli/command/namespaces.py +++ b/client/python/cli/command/namespaces.py @@ -26,12 +26,7 @@ from cli.command import Command from cli.constants import Subcommands, Arguments, UNIT_SEPARATOR from cli.options.option_tree import Argument -from polaris.catalog import ( - IcebergCatalogAPI, - CreateNamespaceRequest, - ApiClient, - Configuration, -) +from polaris.catalog import IcebergCatalogAPI, CreateNamespaceRequest, ApiClient, Configuration from polaris.management import PolarisDefaultApi diff --git a/client/python/cli/command/principal_roles.py b/client/python/cli/command/principal_roles.py index 8dd48bb744..f9685e38eb 100644 --- a/client/python/cli/command/principal_roles.py +++ b/client/python/cli/command/principal_roles.py @@ -24,13 +24,8 @@ from cli.command import Command from cli.constants import Subcommands, Arguments from cli.options.option_tree import Argument -from polaris.management import ( - PolarisDefaultApi, - CreatePrincipalRoleRequest, - PrincipalRole, - UpdatePrincipalRoleRequest, - GrantPrincipalRoleRequest, -) +from polaris.management import PolarisDefaultApi, CreatePrincipalRoleRequest, PrincipalRole, UpdatePrincipalRoleRequest, \ + GrantPrincipalRoleRequest @dataclass diff --git a/client/python/cli/constants.py b/client/python/cli/constants.py index 91fff0dc0f..f82ff2caaf 100644 --- a/client/python/cli/constants.py +++ b/client/python/cli/constants.py @@ -48,6 +48,33 @@ class PrincipalType(Enum): SERVICE = "service" +class CatalogConnectionType(Enum): + """ + Represents a ConnectionType for an EXTERNAL catalog -- see ConnectionConfigInfo in the spec + """ + + HADOOP = 'hadoop' + ICEBERG = 'iceberg-rest' + + +class AuthenticationType(Enum): + """ + Represents a AuthenticationType for an EXTERNAL catalog -- see AuthenticationParameters in the spec + """ + + OAUTH = 'oauth' + BEARER = 'bearer' + SIGV4 = 'sigv4' + + +class ServiceIdentityType(Enum): + """ + Represents a Service Identity Type for an EXTERNAL catalog -- see ServiceIdentityInfo in the spec + """ + + AWS_IAM = 'aws_iam' + + class Commands: """ Represents the various commands available in the CLI @@ -102,40 +129,57 @@ class Arguments: These values should be snake_case, but they will get mapped to kebab-case in `Parser.parse` """ - TYPE = "type" - DEFAULT_BASE_LOCATION = "default_base_location" - STORAGE_TYPE = "storage_type" - ALLOWED_LOCATION = "allowed_location" - ROLE_ARN = "role_arn" - EXTERNAL_ID = "external_id" - USER_ARN = "user_arn" - TENANT_ID = "tenant_id" - MULTI_TENANT_APP_NAME = "multi_tenant_app_name" - CONSENT_URL = "consent_url" - SERVICE_ACCOUNT = "service_account" - CATALOG_ROLE = "catalog_role" - CATALOG = "catalog" - PRINCIPAL = "principal" - CLIENT_ID = "client_id" - PRINCIPAL_ROLE = "principal_role" - PROPERTY = "property" - SET_PROPERTY = "set_property" - REMOVE_PROPERTY = "remove_property" - PRIVILEGE = "privilege" - NAMESPACE = "namespace" - TABLE = "table" - VIEW = "view" - CASCADE = "cascade" - CLIENT_SECRET = "client_secret" - ACCESS_TOKEN = "access_token" - HOST = "host" - PORT = "port" - BASE_URL = "base_url" - PARENT = "parent" - LOCATION = "location" - REGION = "region" - PROFILE = "profile" - PROXY = "proxy" + TYPE = 'type' + DEFAULT_BASE_LOCATION = 'default_base_location' + STORAGE_TYPE = 'storage_type' + ALLOWED_LOCATION = 'allowed_location' + ROLE_ARN = 'role_arn' + EXTERNAL_ID = 'external_id' + USER_ARN = 'user_arn' + TENANT_ID = 'tenant_id' + MULTI_TENANT_APP_NAME = 'multi_tenant_app_name' + CONSENT_URL = 'consent_url' + SERVICE_ACCOUNT = 'service_account' + CATALOG_ROLE = 'catalog_role' + CATALOG = 'catalog' + PRINCIPAL = 'principal' + CLIENT_ID = 'client_id' + PRINCIPAL_ROLE = 'principal_role' + PROPERTY = 'property' + SET_PROPERTY = 'set_property' + REMOVE_PROPERTY = 'remove_property' + PRIVILEGE = 'privilege' + NAMESPACE = 'namespace' + TABLE = 'table' + VIEW = 'view' + CASCADE = 'cascade' + CLIENT_SECRET = 'client_secret' + ACCESS_TOKEN = 'access_token' + HOST = 'host' + PORT = 'port' + BASE_URL = 'base_url' + PARENT = 'parent' + LOCATION = 'location' + REGION = 'region' + PROFILE = 'profile' + PROXY = 'proxy' + HADOOP_WAREHOUSE = 'hadoop_warehouse' + ICEBERG_REMOTE_CATALOG_NAME = 'iceberg_remote_catalog_name' + CATALOG_CONNECTION_TYPE = 'catalog_connection_type' + CATALOG_AUTHENTICATION_TYPE = 'catalog_authentication_type' + CATALOG_SERVICE_IDENTITY_TYPE = 'catalog_service_identity_type' + CATALOG_SERVICE_IDENTITY_IAM_ARN = 'catalog_service_identity_iam_arn' + CATALOG_URI = 'catalog_uri' + CATALOG_TOKEN_URI = 'catalog_token_uri' + CATALOG_CLIENT_ID = 'catalog_client_id' + CATALOG_CLIENT_SECRET = 'catalog_client_secret' + CATALOG_CLIENT_SCOPE = 'catalog_client_scope' + CATALOG_BEARER_TOKEN = 'catalog_bearer_token' + CATALOG_ROLE_ARN = 'catalog_role_arn' + CATALOG_ROLE_SESSION_NAME = 'catalog_role_session_name' + CATALOG_EXTERNAL_ID = 'catalog_external_id' + CATALOG_SIGNING_REGION = 'catalog_signing_region' + CATALOG_SIGNING_NAME = 'catalog_signing_name' class Hints: @@ -192,6 +236,41 @@ class Create: class Update: DEFAULT_BASE_LOCATION = "A new default base location for the catalog" + class External: + CATALOG_CONNECTION_TYPE = 'The type of external catalog in [ICEBERG, HADOOP].' + CATALOG_AUTHENTICATION_TYPE = 'The type of authentication in [OAUTH, BEARER, SIGV4]' + CATALOG_SERVICE_IDENTITY_TYPE = 'The type of service identity in [AWS_IAM]' + + CATALOG_SERVICE_IDENTITY_IAM_ARN = ('When using the AWS_IAM service identity type, this is the ARN ' + 'of the IAM user or IAM role Polaris uses to assume roles and ' + 'then access external resources.') + + CATALOG_URI = 'The URI of the external catalog' + HADOOP_WAREHOUSE = 'The warehouse to use when federating to a HADOOP catalog' + ICEBERG_REMOTE_CATALOG_NAME = 'The remote catalog name when federating to an Iceberg REST catalog' + + + CATALOG_TOKEN_URI = '(For authentication type OAUTH) Token server URI' + CATALOG_CLIENT_ID = '(For authentication type OAUTH) oauth client id' + CATALOG_CLIENT_SECRET = '(For authentication type OAUTH) oauth client secret (input-only)' + CATALOG_CLIENT_SCOPE = ('(For authentication type OAUTH) oauth scopes to specify when exchanging ' + 'for a short-lived access token. Multiple can be provided by specifying' + ' this option more than once') + + CATALOG_BEARER_TOKEN = '(For authentication type BEARER) Bearer token (input-only)' + + CATALOG_ROLE_ARN = ('(For authentication type SIGV4) The aws IAM role arn assumed by polaris ' + 'userArn when signing requests') + CATALOG_ROLE_SESSION_NAME = ('(For authentication type SIGV4) The role session name to be used ' + 'by the SigV4 protocol for signing requests') + CATALOG_EXTERNAL_ID = ('(For authentication type SIGV4) An optional external id used to establish ' + 'a trust relationship with AWS in the trust policy') + CATALOG_SIGNING_REGION = ('(For authentication type SIGV4) Region to be used by the SigV4 protocol ' + 'for signing requests') + CATALOG_SIGNING_NAME = ('(For authentication type SIGV4) The service name to be used by the SigV4 ' + 'protocol for signing requests, the default signing name is "execute-api" ' + 'is if not provided') + class Principals: class Create: TYPE = "The type of principal to create in [SERVICE]" diff --git a/client/python/cli/options/option_tree.py b/client/python/cli/options/option_tree.py index aaec9476d8..d8db86899d 100644 --- a/client/python/cli/options/option_tree.py +++ b/client/python/cli/options/option_tree.py @@ -19,16 +19,8 @@ from dataclasses import dataclass, field from typing import List -from cli.constants import ( - StorageType, - CatalogType, - PrincipalType, - Hints, - Commands, - Arguments, - Subcommands, - Actions, -) +from cli.constants import StorageType, CatalogType, PrincipalType, Hints, Commands, Arguments, Subcommands, Actions, \ + CatalogConnectionType, AuthenticationType, ServiceIdentityType @dataclass @@ -84,534 +76,198 @@ class OptionTree: Argument(Arguments.CATALOG_ROLE, str, Hints.CatalogRoles.CATALOG_ROLE), ] + _FEDERATION_ARGS = [ + Argument(Arguments.CATALOG_CONNECTION_TYPE, str, + Hints.Catalogs.External.CATALOG_CONNECTION_TYPE, lower=True, + choices=[ct.value for ct in CatalogConnectionType]), + Argument(Arguments.ICEBERG_REMOTE_CATALOG_NAME, str, + Hints.Catalogs.External.ICEBERG_REMOTE_CATALOG_NAME), + Argument(Arguments.HADOOP_WAREHOUSE, str, + Hints.Catalogs.External.HADOOP_WAREHOUSE), + Argument(Arguments.CATALOG_AUTHENTICATION_TYPE, str, + Hints.Catalogs.External.CATALOG_AUTHENTICATION_TYPE, lower=True, + choices=[at.value for at in AuthenticationType]), + Argument(Arguments.CATALOG_SERVICE_IDENTITY_TYPE, str, + Hints.Catalogs.External.CATALOG_SERVICE_IDENTITY_TYPE, lower=True, + choices=[st.value for st in ServiceIdentityType]), + Argument(Arguments.CATALOG_SERVICE_IDENTITY_IAM_ARN, str, + Hints.Catalogs.External.CATALOG_SERVICE_IDENTITY_IAM_ARN), + Argument(Arguments.CATALOG_URI, str, Hints.Catalogs.External.CATALOG_URI), + Argument(Arguments.CATALOG_TOKEN_URI, str, Hints.Catalogs.External.CATALOG_TOKEN_URI), + Argument(Arguments.CATALOG_CLIENT_ID, str, Hints.Catalogs.External.CATALOG_CLIENT_ID), + Argument(Arguments.CATALOG_CLIENT_SECRET, str, Hints.Catalogs.External.CATALOG_CLIENT_SECRET), + Argument(Arguments.CATALOG_CLIENT_SCOPE, str, + Hints.Catalogs.External.CATALOG_CLIENT_SCOPE, allow_repeats=True), + Argument(Arguments.CATALOG_BEARER_TOKEN, str, Hints.Catalogs.External.CATALOG_BEARER_TOKEN), + Argument(Arguments.CATALOG_ROLE_ARN, str, Hints.Catalogs.External.CATALOG_ROLE_ARN), + Argument(Arguments.CATALOG_ROLE_SESSION_NAME, str, Hints.Catalogs.External.CATALOG_ROLE_SESSION_NAME), + Argument(Arguments.CATALOG_EXTERNAL_ID, str, Hints.Catalogs.External.CATALOG_EXTERNAL_ID), + Argument(Arguments.CATALOG_SIGNING_REGION, str, Hints.Catalogs.External.CATALOG_SIGNING_REGION), + Argument(Arguments.CATALOG_SIGNING_NAME, str, Hints.Catalogs.External.CATALOG_SIGNING_NAME, lower=True) + ] + @staticmethod def get_tree() -> List[Option]: return [ - Option( - Commands.CATALOGS, - "manage catalogs", - children=[ - Option( - Subcommands.CREATE, - args=[ - Argument( - Arguments.TYPE, - str, - Hints.Catalogs.Create.TYPE, - lower=True, - choices=[ct.value for ct in CatalogType], - default=CatalogType.INTERNAL.value, - ), - Argument( - Arguments.STORAGE_TYPE, - str, - Hints.Catalogs.Create.STORAGE_TYPE, - lower=True, - choices=[st.value for st in StorageType], - ), - Argument( - Arguments.DEFAULT_BASE_LOCATION, - str, - Hints.Catalogs.Create.DEFAULT_BASE_LOCATION, - ), - Argument( - Arguments.ALLOWED_LOCATION, - str, - Hints.Catalogs.Create.ALLOWED_LOCATION, - allow_repeats=True, - ), - Argument( - Arguments.ROLE_ARN, str, Hints.Catalogs.Create.ROLE_ARN - ), - Argument( - Arguments.REGION, str, Hints.Catalogs.Create.REGION - ), - Argument( - Arguments.EXTERNAL_ID, - str, - Hints.Catalogs.Create.EXTERNAL_ID, - ), - Argument( - Arguments.TENANT_ID, - str, - Hints.Catalogs.Create.TENANT_ID, - ), - Argument( - Arguments.MULTI_TENANT_APP_NAME, - str, - Hints.Catalogs.Create.MULTI_TENANT_APP_NAME, - ), - Argument( - Arguments.CONSENT_URL, - str, - Hints.Catalogs.Create.CONSENT_URL, - ), - Argument( - Arguments.SERVICE_ACCOUNT, - str, - Hints.Catalogs.Create.SERVICE_ACCOUNT, - ), - Argument( - Arguments.PROPERTY, - str, - Hints.PROPERTY, - allow_repeats=True, - ), - ], - input_name=Arguments.CATALOG, - ), - Option(Subcommands.DELETE, input_name=Arguments.CATALOG), - Option(Subcommands.GET, input_name=Arguments.CATALOG), - Option( - Subcommands.LIST, - args=[ - Argument( - Arguments.PRINCIPAL_ROLE, - str, - Hints.PrincipalRoles.PRINCIPAL_ROLE, - ) - ], - ), - Option( - Subcommands.UPDATE, - args=[ - Argument( - Arguments.DEFAULT_BASE_LOCATION, - str, - Hints.Catalogs.Update.DEFAULT_BASE_LOCATION, - ), - Argument( - Arguments.ALLOWED_LOCATION, - str, - Hints.Catalogs.Create.ALLOWED_LOCATION, - allow_repeats=True, - ), - Argument( - Arguments.REGION, str, Hints.Catalogs.Create.REGION - ), - Argument( - Arguments.SET_PROPERTY, - str, - Hints.SET_PROPERTY, - allow_repeats=True, - ), - Argument( - Arguments.REMOVE_PROPERTY, - str, - Hints.REMOVE_PROPERTY, - allow_repeats=True, - ), - ], - input_name=Arguments.CATALOG, - ), - ], - ), - Option( - Commands.PRINCIPALS, - "manage principals", - children=[ - Option( - Subcommands.CREATE, - args=[ - Argument( - Arguments.TYPE, - str, - Hints.Principals.Create.TYPE, - lower=True, - choices=[pt.value for pt in PrincipalType], - default=PrincipalType.SERVICE.value, - ), - Argument( - Arguments.PROPERTY, - str, - Hints.PROPERTY, - allow_repeats=True, - ), - ], - input_name=Arguments.PRINCIPAL, - ), - Option(Subcommands.DELETE, input_name=Arguments.PRINCIPAL), - Option(Subcommands.GET, input_name=Arguments.PRINCIPAL), - Option(Subcommands.LIST), - Option( - Subcommands.ROTATE_CREDENTIALS, input_name=Arguments.PRINCIPAL - ), - Option( - Subcommands.UPDATE, - args=[ - Argument( - Arguments.SET_PROPERTY, - str, - Hints.SET_PROPERTY, - allow_repeats=True, - ), - Argument( - Arguments.REMOVE_PROPERTY, - str, - Hints.REMOVE_PROPERTY, - allow_repeats=True, - ), - ], - input_name=Arguments.PRINCIPAL, - ), - Option(Subcommands.ACCESS, input_name=Arguments.PRINCIPAL), - ], - ), - Option( - Commands.PRINCIPAL_ROLES, - "manage principal roles", - children=[ - Option( - Subcommands.CREATE, - args=[ - Argument( - Arguments.PROPERTY, - str, - Hints.PROPERTY, - allow_repeats=True, - ) - ], - input_name=Arguments.PRINCIPAL_ROLE, - ), - Option(Subcommands.DELETE, input_name=Arguments.PRINCIPAL_ROLE), - Option(Subcommands.GET, input_name=Arguments.PRINCIPAL_ROLE), - Option( - Subcommands.LIST, - hint=Hints.PrincipalRoles.LIST, - args=[ - Argument( - Arguments.CATALOG_ROLE, - str, - Hints.PrincipalRoles.List.CATALOG_ROLE, - ), - Argument( - Arguments.PRINCIPAL, - str, - Hints.PrincipalRoles.List.PRINCIPAL_NAME, - ), - ], - ), - Option( - Subcommands.UPDATE, - args=[ - Argument( - Arguments.SET_PROPERTY, - str, - Hints.SET_PROPERTY, - allow_repeats=True, - ), - Argument( - Arguments.REMOVE_PROPERTY, - str, - Hints.REMOVE_PROPERTY, - allow_repeats=True, - ), - ], - input_name=Arguments.PRINCIPAL_ROLE, - ), - Option( - Subcommands.GRANT, - hint=Hints.PrincipalRoles.GRANT, - args=[ - Argument( - Arguments.PRINCIPAL, - str, - Hints.PrincipalRoles.Grant.PRINCIPAL, - ) - ], - input_name=Arguments.PRINCIPAL_ROLE, - ), - Option( - Subcommands.REVOKE, - hint=Hints.PrincipalRoles.REVOKE, - args=[ - Argument( - Arguments.PRINCIPAL, - str, - Hints.PrincipalRoles.Revoke.PRINCIPAL, - ) - ], - input_name=Arguments.PRINCIPAL_ROLE, - ), - ], - ), - Option( - Commands.CATALOG_ROLES, - "manage catalog roles", - children=[ - Option( - Subcommands.CREATE, - args=[ - Argument( - Arguments.CATALOG, str, Hints.CatalogRoles.CATALOG_NAME - ), - Argument( - Arguments.PROPERTY, - str, - Hints.PROPERTY, - allow_repeats=True, - ), - ], - input_name=Arguments.CATALOG_ROLE, - ), - Option( - Subcommands.DELETE, - args=[ - Argument( - Arguments.CATALOG, str, Hints.CatalogRoles.CATALOG_NAME - ), - ], - input_name=Arguments.CATALOG_ROLE, - ), - Option( - Subcommands.GET, - args=[ - Argument( - Arguments.CATALOG, str, Hints.CatalogRoles.CATALOG_NAME - ), - ], - input_name=Arguments.CATALOG_ROLE, - ), - Option( - Subcommands.LIST, - hint=Hints.CatalogRoles.LIST, - args=[ - Argument( - Arguments.PRINCIPAL_ROLE, - str, - Hints.PrincipalRoles.PRINCIPAL_ROLE, - ) - ], - input_name=Arguments.CATALOG, - ), - Option( - Subcommands.UPDATE, - args=[ - Argument( - Arguments.CATALOG, str, Hints.CatalogRoles.CATALOG_NAME - ), - Argument( - Arguments.SET_PROPERTY, - str, - Hints.SET_PROPERTY, - allow_repeats=True, - ), - Argument( - Arguments.REMOVE_PROPERTY, - str, - Hints.REMOVE_PROPERTY, - allow_repeats=True, - ), - ], - input_name=Arguments.CATALOG_ROLE, - ), - Option( - Subcommands.GRANT, - hint=Hints.CatalogRoles.GRANT_CATALOG_ROLE, - args=[ - Argument( - Arguments.CATALOG, str, Hints.CatalogRoles.CATALOG_NAME - ), - Argument( - Arguments.PRINCIPAL_ROLE, - str, - Hints.CatalogRoles.CATALOG_ROLE, - ), - ], - input_name=Arguments.CATALOG_ROLE, - ), - Option( - Subcommands.REVOKE, - hint=Hints.CatalogRoles.GRANT_CATALOG_ROLE, - args=[ - Argument( - Arguments.CATALOG, str, Hints.CatalogRoles.CATALOG_NAME - ), - Argument( - Arguments.PRINCIPAL_ROLE, - str, - Hints.CatalogRoles.CATALOG_ROLE, - ), - ], - input_name=Arguments.CATALOG_ROLE, - ), - ], - ), - Option( - Commands.PRIVILEGES, - "manage privileges for a catalog role", - children=[ - Option(Subcommands.LIST, args=OptionTree._CATALOG_ROLE_AND_CATALOG), - Option( - Subcommands.CATALOG, - children=[ - Option( - Actions.GRANT, - args=OptionTree._CATALOG_ROLE_AND_CATALOG, - input_name=Arguments.PRIVILEGE, - ), - Option( - Actions.REVOKE, - args=[ - Argument( - Arguments.CASCADE, bool, Hints.Grant.CASCADE - ) - ] - + OptionTree._CATALOG_ROLE_AND_CATALOG, - input_name=Arguments.PRIVILEGE, - ), - ], - ), - Option( - Subcommands.NAMESPACE, - children=[ - Option( - Actions.GRANT, - args=[ - Argument( - Arguments.NAMESPACE, str, Hints.Grant.NAMESPACE - ) - ] - + OptionTree._CATALOG_ROLE_AND_CATALOG, - input_name=Arguments.PRIVILEGE, - ), - Option( - Actions.REVOKE, - args=[ - Argument( - Arguments.NAMESPACE, str, Hints.Grant.NAMESPACE - ), - Argument( - Arguments.CASCADE, bool, Hints.Grant.CASCADE - ), - ] - + OptionTree._CATALOG_ROLE_AND_CATALOG, - input_name=Arguments.PRIVILEGE, - ), - ], - ), - Option( - Subcommands.TABLE, - children=[ - Option( - Actions.GRANT, - args=[ - Argument( - Arguments.NAMESPACE, str, Hints.Grant.NAMESPACE - ), - Argument(Arguments.TABLE, str, Hints.Grant.TABLE), - ] - + OptionTree._CATALOG_ROLE_AND_CATALOG, - input_name=Arguments.PRIVILEGE, - ), - Option( - Actions.REVOKE, - args=[ - Argument( - Arguments.NAMESPACE, str, Hints.Grant.NAMESPACE - ), - Argument(Arguments.TABLE, str, Hints.Grant.TABLE), - Argument( - Arguments.CASCADE, bool, Hints.Grant.CASCADE - ), - ] - + OptionTree._CATALOG_ROLE_AND_CATALOG, - input_name=Arguments.PRIVILEGE, - ), - ], - ), - Option( - Subcommands.VIEW, - children=[ - Option( - Actions.GRANT, - args=[ - Argument( - Arguments.NAMESPACE, str, Hints.Grant.NAMESPACE - ), - Argument(Arguments.VIEW, str, Hints.Grant.VIEW), - ] - + OptionTree._CATALOG_ROLE_AND_CATALOG, - input_name=Arguments.PRIVILEGE, - ), - Option( - Actions.REVOKE, - args=[ - Argument( - Arguments.NAMESPACE, str, Hints.Grant.NAMESPACE - ), - Argument(Arguments.VIEW, str, Hints.Grant.VIEW), - Argument( - Arguments.CASCADE, bool, Hints.Grant.CASCADE - ), - ] - + OptionTree._CATALOG_ROLE_AND_CATALOG, - input_name=Arguments.PRIVILEGE, - ), - ], - ), - ], - ), - Option( - Commands.NAMESPACES, - "manage namespaces", - children=[ - Option( - Subcommands.CREATE, - args=[ - Argument( - Arguments.CATALOG, str, Hints.CatalogRoles.CATALOG_NAME - ), - Argument( - Arguments.LOCATION, str, Hints.Namespaces.LOCATION - ), - Argument( - Arguments.PROPERTY, - str, - Hints.PROPERTY, - allow_repeats=True, - ), - ], - input_name=Arguments.NAMESPACE, - ), - Option( - Subcommands.LIST, - args=[ - Argument( - Arguments.CATALOG, str, Hints.CatalogRoles.CATALOG_NAME - ), - Argument(Arguments.PARENT, str, Hints.Namespaces.PARENT), - ], - ), - Option( - Subcommands.DELETE, - args=[ - Argument( - Arguments.CATALOG, str, Hints.CatalogRoles.CATALOG_NAME - ) - ], - input_name=Arguments.NAMESPACE, - ), - Option( - Subcommands.GET, - args=[ - Argument( - Arguments.CATALOG, str, Hints.CatalogRoles.CATALOG_NAME - ) - ], - input_name=Arguments.NAMESPACE, - ), - ], - ), - Option( - Commands.PROFILES, - "manage profiles", - children=[ - Option(Subcommands.CREATE, input_name=Arguments.PROFILE), - Option(Subcommands.DELETE, input_name=Arguments.PROFILE), - Option(Subcommands.UPDATE, input_name=Arguments.PROFILE), - Option(Subcommands.GET, input_name=Arguments.PROFILE), - Option(Subcommands.LIST), - ], - ), + Option(Commands.CATALOGS, 'manage catalogs', children=[ + Option(Subcommands.CREATE, args=[ + Argument(Arguments.TYPE, str, Hints.Catalogs.Create.TYPE, lower=True, + choices=[ct.value for ct in CatalogType], default=CatalogType.INTERNAL.value), + Argument(Arguments.STORAGE_TYPE, str, Hints.Catalogs.Create.STORAGE_TYPE, lower=True, + choices=[st.value for st in StorageType]), + Argument(Arguments.DEFAULT_BASE_LOCATION, str, Hints.Catalogs.Create.DEFAULT_BASE_LOCATION), + Argument(Arguments.ALLOWED_LOCATION, str, Hints.Catalogs.Create.ALLOWED_LOCATION, + allow_repeats=True), + Argument(Arguments.ROLE_ARN, str, Hints.Catalogs.Create.ROLE_ARN), + Argument(Arguments.REGION, str, Hints.Catalogs.Create.REGION), + Argument(Arguments.EXTERNAL_ID, str, Hints.Catalogs.Create.EXTERNAL_ID), + Argument(Arguments.TENANT_ID, str, Hints.Catalogs.Create.TENANT_ID), + Argument(Arguments.MULTI_TENANT_APP_NAME, str, Hints.Catalogs.Create.MULTI_TENANT_APP_NAME), + Argument(Arguments.CONSENT_URL, str, Hints.Catalogs.Create.CONSENT_URL), + Argument(Arguments.SERVICE_ACCOUNT, str, Hints.Catalogs.Create.SERVICE_ACCOUNT), + Argument(Arguments.PROPERTY, str, Hints.PROPERTY, allow_repeats=True), + ] + OptionTree._FEDERATION_ARGS, input_name=Arguments.CATALOG), + Option(Subcommands.DELETE, input_name=Arguments.CATALOG), + Option(Subcommands.GET, input_name=Arguments.CATALOG), + Option(Subcommands.LIST, args=[ + Argument(Arguments.PRINCIPAL_ROLE, str, Hints.PrincipalRoles.PRINCIPAL_ROLE) + ]), + Option(Subcommands.UPDATE, args=[ + Argument(Arguments.DEFAULT_BASE_LOCATION, str, Hints.Catalogs.Update.DEFAULT_BASE_LOCATION), + Argument(Arguments.ALLOWED_LOCATION, str, Hints.Catalogs.Create.ALLOWED_LOCATION, + allow_repeats=True), + Argument(Arguments.REGION, str, Hints.Catalogs.Create.REGION), + Argument(Arguments.SET_PROPERTY, str, Hints.SET_PROPERTY, allow_repeats=True), + Argument(Arguments.REMOVE_PROPERTY, str, Hints.REMOVE_PROPERTY, allow_repeats=True), + ], input_name=Arguments.CATALOG) + ]), + Option(Commands.PRINCIPALS, 'manage principals', children=[ + Option(Subcommands.CREATE, args=[ + Argument(Arguments.TYPE, str, Hints.Principals.Create.TYPE, lower=True, + choices=[pt.value for pt in PrincipalType], default=PrincipalType.SERVICE.value), + Argument(Arguments.PROPERTY, str, Hints.PROPERTY, allow_repeats=True) + ], input_name=Arguments.PRINCIPAL), + Option(Subcommands.DELETE, input_name=Arguments.PRINCIPAL), + Option(Subcommands.GET, input_name=Arguments.PRINCIPAL), + Option(Subcommands.LIST), + Option(Subcommands.ROTATE_CREDENTIALS, input_name=Arguments.PRINCIPAL), + Option(Subcommands.UPDATE, args=[ + Argument(Arguments.SET_PROPERTY, str, Hints.SET_PROPERTY, allow_repeats=True), + Argument(Arguments.REMOVE_PROPERTY, str, Hints.REMOVE_PROPERTY, allow_repeats=True), + ], input_name=Arguments.PRINCIPAL), + Option(Subcommands.ACCESS, input_name=Arguments.PRINCIPAL), + ]), + Option(Commands.PRINCIPAL_ROLES, 'manage principal roles', children=[ + Option(Subcommands.CREATE, args=[ + Argument(Arguments.PROPERTY, str, Hints.PROPERTY, allow_repeats=True) + ], input_name=Arguments.PRINCIPAL_ROLE), + Option(Subcommands.DELETE, input_name=Arguments.PRINCIPAL_ROLE), + Option(Subcommands.GET, input_name=Arguments.PRINCIPAL_ROLE), + Option(Subcommands.LIST, hint=Hints.PrincipalRoles.LIST, args=[ + Argument(Arguments.CATALOG_ROLE, str, Hints.PrincipalRoles.List.CATALOG_ROLE), + Argument(Arguments.PRINCIPAL, str, Hints.PrincipalRoles.List.PRINCIPAL_NAME) + ]), + Option(Subcommands.UPDATE, args=[ + Argument(Arguments.SET_PROPERTY, str, Hints.SET_PROPERTY, allow_repeats=True), + Argument(Arguments.REMOVE_PROPERTY, str, Hints.REMOVE_PROPERTY, allow_repeats=True), + ], input_name=Arguments.PRINCIPAL_ROLE), + Option(Subcommands.GRANT, hint=Hints.PrincipalRoles.GRANT, args=[ + Argument(Arguments.PRINCIPAL, str, Hints.PrincipalRoles.Grant.PRINCIPAL) + ], input_name=Arguments.PRINCIPAL_ROLE), + Option(Subcommands.REVOKE, hint=Hints.PrincipalRoles.REVOKE, args=[ + Argument(Arguments.PRINCIPAL, str, Hints.PrincipalRoles.Revoke.PRINCIPAL) + ], input_name=Arguments.PRINCIPAL_ROLE) + ]), + Option(Commands.CATALOG_ROLES, 'manage catalog roles', children=[ + Option(Subcommands.CREATE, args=[ + Argument(Arguments.CATALOG, str, Hints.CatalogRoles.CATALOG_NAME), + Argument(Arguments.PROPERTY, str, Hints.PROPERTY, allow_repeats=True) + ], input_name=Arguments.CATALOG_ROLE), + Option(Subcommands.DELETE, args=[ + Argument(Arguments.CATALOG, str, Hints.CatalogRoles.CATALOG_NAME), + ], input_name=Arguments.CATALOG_ROLE), + Option(Subcommands.GET, args=[ + Argument(Arguments.CATALOG, str, Hints.CatalogRoles.CATALOG_NAME), + ], input_name=Arguments.CATALOG_ROLE), + Option(Subcommands.LIST, hint=Hints.CatalogRoles.LIST, args=[ + Argument(Arguments.PRINCIPAL_ROLE, str, Hints.PrincipalRoles.PRINCIPAL_ROLE) + ], input_name=Arguments.CATALOG), + Option(Subcommands.UPDATE, args=[ + Argument(Arguments.CATALOG, str, Hints.CatalogRoles.CATALOG_NAME), + Argument(Arguments.SET_PROPERTY, str, Hints.SET_PROPERTY, allow_repeats=True), + Argument(Arguments.REMOVE_PROPERTY, str, Hints.REMOVE_PROPERTY, allow_repeats=True), + ], input_name=Arguments.CATALOG_ROLE), + Option(Subcommands.GRANT, hint=Hints.CatalogRoles.GRANT_CATALOG_ROLE, args=[ + Argument(Arguments.CATALOG, str, Hints.CatalogRoles.CATALOG_NAME), + Argument(Arguments.PRINCIPAL_ROLE, str, Hints.CatalogRoles.CATALOG_ROLE) + ], input_name=Arguments.CATALOG_ROLE), + Option(Subcommands.REVOKE, hint=Hints.CatalogRoles.GRANT_CATALOG_ROLE, args=[ + Argument(Arguments.CATALOG, str, Hints.CatalogRoles.CATALOG_NAME), + Argument(Arguments.PRINCIPAL_ROLE, str, Hints.CatalogRoles.CATALOG_ROLE) + ], input_name=Arguments.CATALOG_ROLE) + ]), + Option(Commands.PRIVILEGES, 'manage privileges for a catalog role', children=[ + Option(Subcommands.LIST, args=OptionTree._CATALOG_ROLE_AND_CATALOG), + Option(Subcommands.CATALOG, children=[ + Option(Actions.GRANT, args=OptionTree._CATALOG_ROLE_AND_CATALOG, input_name=Arguments.PRIVILEGE), + Option(Actions.REVOKE, args=[ + Argument(Arguments.CASCADE, bool, Hints.Grant.CASCADE) + ] + OptionTree._CATALOG_ROLE_AND_CATALOG, input_name=Arguments.PRIVILEGE), + ]), + Option(Subcommands.NAMESPACE, children=[ + Option(Actions.GRANT, args=[ + Argument(Arguments.NAMESPACE, str, Hints.Grant.NAMESPACE) + ] + OptionTree._CATALOG_ROLE_AND_CATALOG, input_name=Arguments.PRIVILEGE), + Option(Actions.REVOKE, args=[ + Argument(Arguments.NAMESPACE, str, Hints.Grant.NAMESPACE), + Argument(Arguments.CASCADE, bool, Hints.Grant.CASCADE) + ] + OptionTree._CATALOG_ROLE_AND_CATALOG, input_name=Arguments.PRIVILEGE), + ]), + Option(Subcommands.TABLE, children=[ + Option(Actions.GRANT, args=[ + Argument(Arguments.NAMESPACE, str, Hints.Grant.NAMESPACE), + Argument(Arguments.TABLE, str, Hints.Grant.TABLE) + ] + OptionTree._CATALOG_ROLE_AND_CATALOG, input_name=Arguments.PRIVILEGE), + Option(Actions.REVOKE, args=[ + Argument(Arguments.NAMESPACE, str, Hints.Grant.NAMESPACE), + Argument(Arguments.TABLE, str, Hints.Grant.TABLE), + Argument(Arguments.CASCADE, bool, Hints.Grant.CASCADE) + ] + OptionTree._CATALOG_ROLE_AND_CATALOG, input_name=Arguments.PRIVILEGE), + ]), + Option(Subcommands.VIEW, children=[ + Option(Actions.GRANT, args=[ + Argument(Arguments.NAMESPACE, str, Hints.Grant.NAMESPACE), + Argument(Arguments.VIEW, str, Hints.Grant.VIEW) + ] + OptionTree._CATALOG_ROLE_AND_CATALOG, input_name=Arguments.PRIVILEGE), + Option(Actions.REVOKE, args=[ + Argument(Arguments.NAMESPACE, str, Hints.Grant.NAMESPACE), + Argument(Arguments.VIEW, str, Hints.Grant.VIEW), + Argument(Arguments.CASCADE, bool, Hints.Grant.CASCADE) + ] + OptionTree._CATALOG_ROLE_AND_CATALOG, input_name=Arguments.PRIVILEGE), + ]) + ]), + Option(Commands.NAMESPACES, 'manage namespaces', children=[ + Option(Subcommands.CREATE, args=[ + Argument(Arguments.CATALOG, str, Hints.CatalogRoles.CATALOG_NAME), + Argument(Arguments.LOCATION, str, Hints.Namespaces.LOCATION), + Argument(Arguments.PROPERTY, str, Hints.PROPERTY, allow_repeats=True) + ], input_name=Arguments.NAMESPACE), + Option(Subcommands.LIST, args=[ + Argument(Arguments.CATALOG, str, Hints.CatalogRoles.CATALOG_NAME), + Argument(Arguments.PARENT, str, Hints.Namespaces.PARENT) + ]), + Option(Subcommands.DELETE, args=[ + Argument(Arguments.CATALOG, str, Hints.CatalogRoles.CATALOG_NAME) + ], input_name=Arguments.NAMESPACE), + Option(Subcommands.GET, args=[ + Argument(Arguments.CATALOG, str, Hints.CatalogRoles.CATALOG_NAME) + ], input_name=Arguments.NAMESPACE), + ]), + Option(Commands.PROFILES, 'manage profiles', children=[ + Option(Subcommands.CREATE, input_name=Arguments.PROFILE), + Option(Subcommands.DELETE, input_name=Arguments.PROFILE), + Option(Subcommands.UPDATE, input_name=Arguments.PROFILE), + Option(Subcommands.GET, input_name=Arguments.PROFILE), + Option(Subcommands.LIST), + ]) ] diff --git a/client/python/cli/options/parser.py b/client/python/cli/options/parser.py index 26f5c8dfcb..0c915261c2 100644 --- a/client/python/cli/options/parser.py +++ b/client/python/cli/options/parser.py @@ -86,10 +86,8 @@ def add_arguments(parser, args: List[Argument]): kwargs["default"] = arg.default if arg.type is bool: - del kwargs["type"] - parser.add_argument( - arg.get_flag_name(), **kwargs, action="store_true" - ) + del kwargs['type'] + parser.add_argument(arg.get_flag_name(), **kwargs, action='store_true') elif arg.allow_repeats: parser.add_argument(arg.get_flag_name(), **kwargs, action="append") else: @@ -156,8 +154,8 @@ def parse_args(self, args=None, namespace=None): if help_index < float("inf"): tree_str = self._get_tree_str(args[:help_index]) if tree_str: - print(f"input: polaris {' '.join(args)}") - print("options:") + print(f'input: polaris {" ".join(args)}') + print('options:') print(tree_str) print("\n") self.print_usage() diff --git a/client/python/cli/polaris_cli.py b/client/python/cli/polaris_cli.py index 83341ada41..47e8038650 100644 --- a/client/python/cli/polaris_cli.py +++ b/client/python/cli/polaris_cli.py @@ -158,11 +158,9 @@ def _get_client_builder(options): # Authenticate accordingly if options.base_url: if options.host is not None or options.port is not None: - raise Exception( - f"Please provide either {Argument.to_flag_name(Arguments.BASE_URL)} or" - f" {Argument.to_flag_name(Arguments.HOST)} &" - f" {Argument.to_flag_name(Arguments.PORT)}, but not both" - ) + raise Exception(f'Please provide either {Argument.to_flag_name(Arguments.BASE_URL)} or' + f' {Argument.to_flag_name(Arguments.HOST)} &' + f' {Argument.to_flag_name(Arguments.PORT)}, but not both') polaris_management_url = f"{options.base_url}/api/management/v1" polaris_catalog_url = f"{options.base_url}/api/catalog/v1" diff --git a/client/python/test/test_cli_parsing.py b/client/python/test/test_cli_parsing.py index bc4f59afeb..715e2e3af8 100644 --- a/client/python/test/test_cli_parsing.py +++ b/client/python/test/test_cli_parsing.py @@ -504,6 +504,85 @@ def get(obj, arg_string): (2, 'grant.namespace'): ['a', 'b', 'c'], (2, 'grant.view_name'): 'v', }) + check_arguments( + mock_execute(['catalogs', 'create', 'my-catalog', '--type', 'external', + '--storage-type', 'gcs', '--default-base-location', 'dbl', + '--catalog-connection-type', 'hadoop', '--hadoop-warehouse', 'h', + '--catalog-uri', 'u', '--catalog-authentication-type', 'bearer', + '--catalog-bearer-token', 'b']), + 'create_catalog', { + (0, 'catalog.name'): 'my-catalog', + (0, 'catalog.type'): 'EXTERNAL', + (0, 'catalog.connection_config_info.connection_type'): 'HADOOP', + (0, 'catalog.connection_config_info.warehouse'): 'h', + (0, 'catalog.connection_config_info.uri'): 'u', + }) + check_arguments( + mock_execute(['catalogs', 'create', 'my-catalog', '--type', 'external', + '--storage-type', 'gcs', '--default-base-location', 'dbl', + '--catalog-connection-type', 'iceberg-rest', '--iceberg-remote-catalog-name', 'i', + '--catalog-uri', 'u', '--catalog-authentication-type', 'bearer', + '--catalog-bearer-token', 'b']), + 'create_catalog', { + (0, 'catalog.name'): 'my-catalog', + (0, 'catalog.type'): 'EXTERNAL', + (0, 'catalog.connection_config_info.connection_type'): 'ICEBERG_REST', + (0, 'catalog.connection_config_info.remote_catalog_name'): 'i', + (0, 'catalog.connection_config_info.uri'): 'u', + }) + check_arguments( + mock_execute(['catalogs', 'create', 'my-catalog', '--type', 'external', + '--storage-type', 'gcs', '--default-base-location', 'dbl', + '--catalog-connection-type', 'hadoop', '--hadoop-warehouse', 'h', + '--catalog-authentication-type', 'oauth', + '--catalog-token-uri', 'u', '--catalog-client-id', 'i', + '--catalog-client-secret', 'k', '--catalog-client-scope', 's1', + '--catalog-client-scope', 's2']), + 'create_catalog', { + (0, 'catalog.name'): 'my-catalog', + (0, 'catalog.type'): 'EXTERNAL', + (0, 'catalog.connection_config_info.connection_type'): 'HADOOP', + (0, 'catalog.connection_config_info.warehouse'): 'h', + (0, 'catalog.connection_config_info.authentication_parameters.authentication_type'): 'OAUTH', + (0, 'catalog.connection_config_info.authentication_parameters.token_uri'): 'u', + (0, 'catalog.connection_config_info.authentication_parameters.client_id'): 'i', + (0, 'catalog.connection_config_info.authentication_parameters.scopes'): ['s1', 's2'], + }) + check_arguments( + mock_execute(['catalogs', 'create', 'my-catalog', '--type', 'external', + '--storage-type', 'gcs', '--default-base-location', 'dbl', + '--catalog-connection-type', 'iceberg-rest', '--iceberg-remote-catalog-name', 'i', + '--catalog-uri', 'u', '--catalog-authentication-type', 'sigv4', + '--catalog-role-arn', 'a', '--catalog-signing-region', 's']), + 'create_catalog', { + (0, 'catalog.name'): 'my-catalog', + (0, 'catalog.type'): 'EXTERNAL', + (0, 'catalog.connection_config_info.connection_type'): 'ICEBERG_REST', + (0, 'catalog.connection_config_info.remote_catalog_name'): 'i', + (0, 'catalog.connection_config_info.uri'): 'u', + (0, 'catalog.connection_config_info.authentication_parameters.role_arn'): 'a', + (0, 'catalog.connection_config_info.authentication_parameters.signing_region'): 's', + }) + check_arguments( + mock_execute(['catalogs', 'create', 'my-catalog', '--type', 'external', + '--storage-type', 'gcs', '--default-base-location', 'dbl', + '--catalog-connection-type', 'iceberg-rest', '--iceberg-remote-catalog-name', 'i', + '--catalog-uri', 'u', '--catalog-authentication-type', 'sigv4', + '--catalog-role-arn', 'a', '--catalog-signing-region', 's', + '--catalog-role-session-name', 'n', '--catalog-external-id', 'i', + '--catalog-signing-name', 'g']), + 'create_catalog', { + (0, 'catalog.name'): 'my-catalog', + (0, 'catalog.type'): 'EXTERNAL', + (0, 'catalog.connection_config_info.connection_type'): 'ICEBERG_REST', + (0, 'catalog.connection_config_info.remote_catalog_name'): 'i', + (0, 'catalog.connection_config_info.uri'): 'u', + (0, 'catalog.connection_config_info.authentication_parameters.role_arn'): 'a', + (0, 'catalog.connection_config_info.authentication_parameters.signing_region'): 's', + (0, 'catalog.connection_config_info.authentication_parameters.role_session_name'): 'n', + (0, 'catalog.connection_config_info.authentication_parameters.external_id'): 'i', + (0, 'catalog.connection_config_info.authentication_parameters.signing_name'): 'g', + }) if __name__ == '__main__': diff --git a/getting-started/spark/notebooks/SparkPolaris.ipynb b/getting-started/spark/notebooks/SparkPolaris.ipynb index c015235406..7168efaa68 100644 --- a/getting-started/spark/notebooks/SparkPolaris.ipynb +++ b/getting-started/spark/notebooks/SparkPolaris.ipynb @@ -102,7 +102,7 @@ " try:\n", " api.create_catalog_role(catalog_name=catalog.name, create_catalog_role_request=CreateCatalogRoleRequest(catalog_role=catalog_role))\n", " return api.get_catalog_role(catalog_name=catalog.name, catalog_role_name=role_name)\n", - " except ApiException as e:\n", + " except ApiException:\n", " return api.get_catalog_role(catalog_name=catalog.name, catalog_role_name=role_name)\n", " else:\n", " raise e\n", @@ -113,7 +113,7 @@ " try:\n", " api.create_principal_role(CreatePrincipalRoleRequest(principal_role=principal_role))\n", " return api.get_principal_role(principal_role_name=role_name)\n", - " except ApiException as e:\n", + " except ApiException:\n", " return api.get_principal_role(principal_role_name=role_name)\n" ] }, @@ -445,7 +445,6 @@ "outputs": [], "source": [ "import codecs\n", - "import json\n", "from IPython.display import display, JSON\n", "\n", "def format_namespace(namespace):\n", diff --git a/plugins/spark/v3.5/getting-started/notebooks/SparkPolaris.ipynb b/plugins/spark/v3.5/getting-started/notebooks/SparkPolaris.ipynb index 15283dc3f7..1c3803d7b0 100644 --- a/plugins/spark/v3.5/getting-started/notebooks/SparkPolaris.ipynb +++ b/plugins/spark/v3.5/getting-started/notebooks/SparkPolaris.ipynb @@ -16,7 +16,6 @@ "metadata": {}, "outputs": [], "source": [ - "from polaris.catalog.api.iceberg_catalog_api import IcebergCatalogAPI\n", "from polaris.catalog.api.iceberg_o_auth2_api import IcebergOAuth2API\n", "from polaris.catalog.api_client import ApiClient as CatalogApiClient\n", "from polaris.catalog.api_client import Configuration as CatalogApiClientConfiguration\n", @@ -113,7 +112,7 @@ " try:\n", " api.create_catalog_role(catalog_name=catalog.name, create_catalog_role_request=CreateCatalogRoleRequest(catalog_role=catalog_role))\n", " return api.get_catalog_role(catalog_name=catalog.name, catalog_role_name=role_name)\n", - " except ApiException as e:\n", + " except ApiException:\n", " return api.get_catalog_role(catalog_name=catalog.name, catalog_role_name=role_name)\n", " else:\n", " raise e\n", @@ -124,7 +123,7 @@ " try:\n", " api.create_principal_role(CreatePrincipalRoleRequest(principal_role=principal_role))\n", " return api.get_principal_role(principal_role_name=role_name)\n", - " except ApiException as e:\n", + " except ApiException:\n", " return api.get_principal_role(principal_role_name=role_name)\n" ] }, diff --git a/regtests/t_cli/src/test_cli.py b/regtests/t_cli/src/test_cli.py index d60560f202..ee6bea7147 100644 --- a/regtests/t_cli/src/test_cli.py +++ b/regtests/t_cli/src/test_cli.py @@ -16,8 +16,6 @@ # specific language governing permissions and limitations # under the License. # -import contextlib -import io import json import os import random @@ -157,7 +155,7 @@ def test_quickstart_flow(): f'test_cli_catalog_{SALT}', '--catalog-role', f'test_cli_c_role_{SALT}', - f'CATALOG_MANAGE_CONTENT' + 'CATALOG_MANAGE_CONTENT' ), checker=lambda s: s == '') # User now has catalog access: @@ -835,7 +833,7 @@ def test_list_privileges(): f'test_cli_catalog_{SALT}', '--catalog-role', f'test_cli_c_role_{SALT}', - f'TABLE_READ_DATA' + 'TABLE_READ_DATA' ), checker=lambda s: s == '') check_output(root_cli( 'privileges', @@ -847,7 +845,7 @@ def test_list_privileges(): f'test_cli_c_role_{SALT}', '--namespace', f'a_{SALT}', - f'TABLE_WRITE_DATA' + 'TABLE_WRITE_DATA' ), checker=lambda s: s == '') check_output(root_cli( 'privileges', @@ -859,7 +857,7 @@ def test_list_privileges(): f'test_cli_c_role_{SALT}', '--namespace', f'a_{SALT}', - f'TABLE_LIST' + 'TABLE_LIST' ), checker=lambda s: s == '') # List privileges: diff --git a/regtests/t_oauth/test_oauth2_tokens.py b/regtests/t_oauth/test_oauth2_tokens.py index e0019acfd4..3e165d7128 100644 --- a/regtests/t_oauth/test_oauth2_tokens.py +++ b/regtests/t_oauth/test_oauth2_tokens.py @@ -22,7 +22,6 @@ """ import argparse import requests -import urllib def main(base_uri, client_id, client_secret): diff --git a/regtests/t_pyspark/src/conftest.py b/regtests/t_pyspark/src/conftest.py index da14b14a77..a250257af9 100644 --- a/regtests/t_pyspark/src/conftest.py +++ b/regtests/t_pyspark/src/conftest.py @@ -148,7 +148,7 @@ def create_catalog_role(api, catalog, role_name): api.create_catalog_role(catalog_name=catalog.name, create_catalog_role_request=CreateCatalogRoleRequest(catalog_role=catalog_role)) return api.get_catalog_role(catalog_name=catalog.name, catalog_role_name=role_name) - except ApiException as e: + except ApiException: return api.get_catalog_role(catalog_name=catalog.name, catalog_role_name=role_name) else: raise e diff --git a/regtests/t_pyspark/src/iceberg_spark.py b/regtests/t_pyspark/src/iceberg_spark.py index 8cb20591fc..f1bc295a73 100644 --- a/regtests/t_pyspark/src/iceberg_spark.py +++ b/regtests/t_pyspark/src/iceberg_spark.py @@ -21,7 +21,6 @@ import os from typing import Any, Dict, List, Optional, Union -from pyspark.errors import PySparkRuntimeError from pyspark.sql import SparkSession diff --git a/regtests/t_pyspark/src/test_spark_sql_s3_with_privileges.py b/regtests/t_pyspark/src/test_spark_sql_s3_with_privileges.py index 3d31de4649..89c7b5568b 100644 --- a/regtests/t_pyspark/src/test_spark_sql_s3_with_privileges.py +++ b/regtests/t_pyspark/src/test_spark_sql_s3_with_privileges.py @@ -28,10 +28,9 @@ import pytest from py4j.protocol import Py4JJavaError -from botocore.exceptions import ClientError from iceberg_spark import IcebergSparkSession -from polaris.catalog import CreateNamespaceRequest, CreateTableRequest, ModelSchema, StructField +from polaris.catalog import CreateNamespaceRequest, CreateTableRequest, ModelSchema from polaris.catalog.api.iceberg_catalog_api import IcebergCatalogAPI from polaris.catalog.api.iceberg_o_auth2_api import IcebergOAuth2API from polaris.catalog.api_client import ApiClient as CatalogApiClient @@ -353,17 +352,17 @@ def test_spark_creates_table_in_custom_namespace_dir(root_client, snowflake_cata spark.sql('CREATE NAMESPACE db1') spark.sql(f"CREATE NAMESPACE db1.schema LOCATION '{namespace_location}'") spark.sql('USE db1.schema') - spark.sql(f"CREATE TABLE table_in_custom_namespace_location (col1 int, col2 string)") + spark.sql("CREATE TABLE table_in_custom_namespace_location (col1 int, col2 string)") assert spark.sql("SELECT * FROM table_in_custom_namespace_location").count() == 0 # check the metadata and assert the custom namespace location is used entries = spark.sql( - f"SELECT file FROM db1.schema.table_in_custom_namespace_location.metadata_log_entries").collect() + "SELECT file FROM db1.schema.table_in_custom_namespace_location.metadata_log_entries").collect() assert namespace_location in entries[0][0] try: assert spark.sql("SELECT * FROM table_in_custom_namespace_location").count() == 0 # check the metadata and assert the custom namespace location is used entries = spark.sql( - f"SELECT file FROM db1.schema.table_in_custom_namespace_location.metadata_log_entries").collect() + "SELECT file FROM db1.schema.table_in_custom_namespace_location.metadata_log_entries").collect() assert namespace_location in entries[0][0] finally: spark.sql('DROP TABLE table_in_custom_namespace_location PURGE') @@ -1233,7 +1232,7 @@ def create_catalog_role(api, catalog, role_name): api.create_catalog_role(catalog_name=catalog.name, create_catalog_role_request=CreateCatalogRoleRequest(catalog_role=catalog_role)) return api.get_catalog_role(catalog_name=catalog.name, catalog_role_name=role_name) - except ApiException as e: + except ApiException: return api.get_catalog_role(catalog_name=catalog.name, catalog_role_name=role_name) else: raise e @@ -1244,5 +1243,5 @@ def create_principal_role(api, role_name): try: api.create_principal_role(CreatePrincipalRoleRequest(principal_role=principal_role)) return api.get_principal_role(principal_role_name=role_name) - except ApiException as e: + except ApiException: return api.get_principal_role(principal_role_name=role_name) diff --git a/site/content/in-dev/unreleased/command-line-interface.md b/site/content/in-dev/unreleased/command-line-interface.md index 8b53166b0c..cb0d11298d 100644 --- a/site/content/in-dev/unreleased/command-line-interface.md +++ b/site/content/in-dev/unreleased/command-line-interface.md @@ -133,12 +133,30 @@ options: --default-base-location (Required) Default base location of the catalog --allowed-location An allowed location for files tracked by the catalog. Multiple locations can be provided by specifying this option more than once. --role-arn (Required for S3) A role ARN to use when connecting to S3 + --region (Only for S3) The region to use when connecting to S3 --external-id (Only for S3) The external ID to use when connecting to S3 --tenant-id (Required for Azure) A tenant ID to use when connecting to Azure Storage --multi-tenant-app-name (Only for Azure) The app name to use when connecting to Azure Storage --consent-url (Only for Azure) A consent URL granting permissions for the Azure Storage location --service-account (Only for GCS) The service account to use when connecting to GCS --property A key/value pair such as: tag=value. Multiple can be provided by specifying this option more than once + --catalog-connection-type The type of external catalog in [ICEBERG, HADOOP]. + --iceberg-remote-catalog-name The remote catalog name when federating to an Iceberg REST catalog + --hadoop-warehouse The warehouse to use when federating to a HADOOP catalog + --catalog-authentication-type The type of authentication in [OAUTH, BEARER, SIGV4] + --catalog-service-identity-type The type of service identity in [AWS_IAM] + --catalog-service-identity-iam-arn When using the AWS_IAM service identity type, this is the ARN of the IAM user or IAM role Polaris uses to assume roles and then access external resources. + --catalog-uri The URI of the external catalog + --catalog-token-uri (For authentication type OAUTH) Token server URI + --catalog-client-id (For authentication type OAUTH) oauth client id + --catalog-client-secret (For authentication type OAUTH) oauth client secret (input-only) + --catalog-client-scope (For authentication type OAUTH) oauth scopes to specify when exchanging for a short-lived access token. Multiple can be provided by specifying this option more than once + --catalog-bearer-token (For authentication type BEARER) Bearer token (input-only) + --catalog-role-arn (For authentication type SIGV4) The aws IAM role arn assumed by polaris userArn when signing requests + --catalog-role-session-name (For authentication type SIGV4) The role session name to be used by the SigV4 protocol for signing requests + --catalog-external-id (For authentication type SIGV4) An optional external id used to establish a trust relationship with AWS in the trust policy + --catalog-signing-region (For authentication type SIGV4) Region to be used by the SigV4 protocol for signing requests + --catalog-signing-name (For authentication type SIGV4) The service name to be used by the SigV4 protocol for signing requests, the default signing name is "execute-api" is if not provided Positional arguments: catalog ```